From 27643d5e23fe0130d95411bcbaf0c9f613814ebc Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Mon, 13 Dec 2021 18:36:21 +0530 Subject: [PATCH 01/33] YARN-8234. Improve RM system metrics publisher's performance by pushing events to timeline server in batch. --- .../hadoop/yarn/conf/YarnConfiguration.java | 14 ++ .../src/main/resources/yarn-default.xml | 27 +++ .../metrics/TimelineServiceV1Publisher.java | 179 +++++++++++++++++- .../applicationsmanager/TestAMRestart.java | 1 + .../TestCombinedSystemMetricsPublisher.java | 2 + .../metrics/TestSystemMetricsPublisher.java | 2 + 6 files changed, 215 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 57cc247a941cb0..7df41498ef27a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -762,6 +762,20 @@ public static boolean isAclEnabled(Configuration conf) { public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 10; + public static final String RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE = + RM_PREFIX + "system-metrics-publisher.timeline-server-v1.batch-size"; + public static final int + DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE = + 1000; + public static final String RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL = + RM_PREFIX + "system-metrics-publisher.timeline-server-v1.interval-seconds"; + public static final int DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL = + 60; + public static final String RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED = + RM_PREFIX + "system-metrics-publisher.timeline-server-v1.enable-batch"; + public static final boolean DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED = + false; + //RM delegation token related keys public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY = RM_PREFIX + "delegation.key.update-interval"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index b3d86d0760162d..d6bef5b3a956d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1008,6 +1008,33 @@ 10 + + + This setting enables/disables timeline server v1 publisher to publish timeline events in batch. + + yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch + false + + + + + The size of timeline server v1 publisher sending events in one request. + + yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size + 1000 + + + + + When enable batch publishing in timeline server v1, we must avoid that the + publisher waits for a batch to be filled up and hold events in buffer for long + time. So we add another thread which send event's in the buffer periodically. + This config sets the interval of the cyclical sending thread. + + yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds + 60 + + Number of diagnostics/failure messages can be saved in RM for log aggregation. It also defines the number of diagnostics/failure diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java index 23aba4a23b2a00..86576f7b1d5432 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java @@ -18,8 +18,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.client.api.TimelineClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; @@ -59,9 +65,43 @@ public TimelineServiceV1Publisher() { } private TimelineClient client; + private LinkedBlockingQueue entityQueue; + private ExecutorService sendEventThreadPool; + private int dispatcherPoolSize; + private int dispatcherBatchSize; + private int putEventInterval; + private boolean isTimeLineServerBatchEnabled; + private volatile boolean stopped = false; + private PutEventThread putEventThread; + private Object sendEntityLock; @Override protected void serviceInit(Configuration conf) throws Exception { + isTimeLineServerBatchEnabled = + conf.getBoolean( + YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED, + YarnConfiguration.DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED); + if (isTimeLineServerBatchEnabled) { + putEventInterval = + conf.getInt(YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL, + YarnConfiguration.DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL) + * 1000; + dispatcherPoolSize = conf.getInt( + YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE, + YarnConfiguration. + DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE); + dispatcherBatchSize = conf.getInt( + YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE, + YarnConfiguration. + DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE); + putEventThread = new PutEventThread(); + sendEventThreadPool = Executors.newFixedThreadPool(dispatcherPoolSize); + entityQueue = new LinkedBlockingQueue<>(dispatcherBatchSize + 1); + sendEntityLock = new Object(); + LOG.info("Timeline service v1 batch publishing enabled"); + } else { + LOG.info("Timeline service v1 batch publishing disabled"); + } client = TimelineClient.createTimelineClient(); addIfService(client); super.serviceInit(conf); @@ -69,6 +109,35 @@ protected void serviceInit(Configuration conf) throws Exception { new TimelineV1EventHandler()); } + protected void serviceStart() throws Exception { + if (isTimeLineServerBatchEnabled) { + stopped = false; + putEventThread.start(); + } + super.serviceStart(); + } + + protected void serviceStop() throws Exception { + super.serviceStop(); + if (isTimeLineServerBatchEnabled) { + stopped = true; + putEventThread.interrupt(); + try { + putEventThread.join(); + SendEntity task = new SendEntity(); + if (!task.buffer.isEmpty()) { + LOG.info(String.format("Initiating final putEntities, remaining entities left in entityQueue: %d", task.buffer.size())); + sendEventThreadPool.submit(task); + } + } finally { + sendEventThreadPool.shutdown(); + if (!sendEventThreadPool.awaitTermination(3, TimeUnit.SECONDS)) { + sendEventThreadPool.shutdownNow(); + } + } + } + } + @SuppressWarnings("unchecked") @Override public void appCreated(RMApp app, long createdTime) { @@ -257,7 +326,7 @@ public void appAttemptRegistered(RMAppAttempt appAttempt, @SuppressWarnings("unchecked") @Override public void appAttemptFinished(RMAppAttempt appAttempt, - RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) { + RMAppAttemptState appAttemptState, RMApp app, long finishedTime) { TimelineEntity entity = createAppAttemptEntity(appAttempt.getAppAttemptId()); @@ -274,7 +343,7 @@ public void appAttemptFinished(RMAppAttempt appAttempt, eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_INFO, app.getFinalApplicationStatus().toString()); eventInfo.put(AppAttemptMetricsConstants.STATE_INFO, RMServerUtils - .createApplicationAttemptState(appAttemtpState).toString()); + .createApplicationAttemptState(appAttemptState).toString()); if (appAttempt.getMasterContainer() != null) { eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_INFO, appAttempt.getMasterContainer().getId().toString()); @@ -374,16 +443,62 @@ private static TimelineEntity createContainerEntity(ContainerId containerId) { } private void putEntity(TimelineEntity entity) { - try { + if (isTimeLineServerBatchEnabled) { + try { + entityQueue.put(entity); + if (entityQueue.size() > dispatcherBatchSize) { + SendEntity task = null; + synchronized (sendEntityLock) { + if (entityQueue.size() > dispatcherBatchSize) { + task = new SendEntity(); + } + } + if (task != null) { + sendEventThreadPool.submit(task); + } + } + } catch (Exception e) { + LOG.error("Error when publishing entity batch [ " + entity.getEntityType() + "," + + entity.getEntityId() + " ] ", e); + } + } + else { + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Publishing the entity " + entity.getEntityId() + + ", JSON-style content: " + + TimelineUtils.dumpTimelineRecordtoJSON(entity)); + } + client.putEntities(entity); + } catch (Exception e) { + LOG.error("Error when publishing entity [ " + entity.getEntityType() + "," + + entity.getEntityId() + " ] ", e); + } + } + } + + private class SendEntity implements Runnable { + + private ArrayList buffer; + + public SendEntity(){ + buffer = new ArrayList(); + entityQueue.drainTo(buffer); + } + + @Override + public void run() { if (LOG.isDebugEnabled()) { - LOG.debug("Publishing the entity " + entity.getEntityId() - + ", JSON-style content: " - + TimelineUtils.dumpTimelineRecordtoJSON(entity)); + LOG.debug(String.format("Number of timeline entities being sent in batch: %d", buffer.size())); + } + if (buffer.isEmpty()) { + return; + } + try { + client.putEntities(buffer.toArray(new TimelineEntity[0])); + } catch (Exception e) { + LOG.error("Error when publishing entity: ", e); } - client.putEntities(entity); - } catch (Exception e) { - LOG.error("Error when publishing entity [" + entity.getEntityType() + "," - + entity.getEntityId() + "]", e); } } @@ -408,4 +523,48 @@ public void handle(TimelineV1PublishEvent event) { putEntity(event.getEntity()); } } + + private class PutEventThread extends Thread { + public PutEventThread() { + super("PutEventThread"); + } + + @Override + public void run() { + LOG.info("System metrics publisher will put events every " + + String.valueOf(putEventInterval) + " milliseconds"); + while (!stopped && !Thread.currentThread().isInterrupted()) { + if (System.currentTimeMillis() % putEventInterval >= 1000) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + LOG.warn(SystemMetricsPublisher.class.getName() + + " is interrupted. Exiting."); + break; + } + continue; + } + SendEntity task = null; + synchronized (sendEntityLock) { + if (LOG.isDebugEnabled()) { + LOG.debug("Creating SendEntity task in PutEventThread"); + } + task = new SendEntity(); + } + if (task != null) { + sendEventThreadPool.submit(task); + } + try { + // sleep added to avoid multiple SendEntity task within a single interval. + Thread.sleep(1000); + } catch (InterruptedException e) { + LOG.warn(SystemMetricsPublisher.class.getName() + + " is interrupted. Exiting."); + break; + } + } + } + } } + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index 82c534e6aad79e..23332836f80f55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -587,6 +587,7 @@ public void testPreemptedAMRestartOnRMRestart() throws Exception { getConf().set( YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); getConf().setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); + getConf().setInt(YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL, 1); MockRM rm1 = new MockRM(getConf()); MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java index 63f007b45b923e..33b9eecb15f9da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java @@ -203,6 +203,8 @@ private static YarnConfiguration getConf(boolean v1Enabled, MemoryTimelineStore.class, TimelineStore.class); yarnConf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS, MemoryTimelineStateStore.class, TimelineStateStore.class); + yarnConf.setInt(YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL, + 1); } if (v2Enabled) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 3c00bbcdc071e2..a9a57314cc4208 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -88,6 +88,8 @@ public static void setup() throws Exception { conf.setInt( YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE, 2); + conf.setInt(YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL, + 1); timelineServer = new ApplicationHistoryServer(); timelineServer.init(conf); From ab2553d1b0b25c219bd7f5446cc5d1737a6e7cc2 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Fri, 10 Dec 2021 01:36:31 +0900 Subject: [PATCH 02/33] HADOOP-18040. Use maven.test.failure.ignore instead of ignoreTestFailure (#3774) Reviewed-by: Masatake Iwasaki --- hadoop-common-project/hadoop-common/pom.xml | 1 - hadoop-common-project/hadoop-kms/pom.xml | 1 - hadoop-common-project/hadoop-registry/pom.xml | 1 - hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 2 -- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 - hadoop-project/pom.xml | 3 +-- hadoop-tools/hadoop-distcp/pom.xml | 1 - hadoop-tools/hadoop-federation-balance/pom.xml | 1 - 8 files changed, 1 insertion(+), 10 deletions(-) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index bcba2288300f32..a75ab5ecc4569f 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -915,7 +915,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} ${testsThreadCount} false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index 9de8b9caf6e68c..96588a22b94193 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -186,7 +186,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} 1 false 1 diff --git a/hadoop-common-project/hadoop-registry/pom.xml b/hadoop-common-project/hadoop-registry/pom.xml index 171b7229035fbc..725dda50f216bd 100644 --- a/hadoop-common-project/hadoop-registry/pom.xml +++ b/hadoop-common-project/hadoop-registry/pom.xml @@ -231,7 +231,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} false 900 -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index 1916ef0e3b7f6e..a1b3ab1f923976 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -247,7 +247,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} 1 600 @@ -361,7 +360,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} 1 true 600 diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 10d66d055ba359..bc05d850fe8f71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -471,7 +471,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} ${testsThreadCount} false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 6b7e0165ee1cb9..62e047254b7077 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -35,7 +35,7 @@ false - true + true true 9.4.44.v20210927 _ @@ -2126,7 +2126,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} false ${surefire.fork.timeout} ${maven-surefire-plugin.argLine} diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml index 7e5aaebc085132..55738ef808c284 100644 --- a/hadoop-tools/hadoop-distcp/pom.xml +++ b/hadoop-tools/hadoop-distcp/pom.xml @@ -128,7 +128,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} 1 false 600 diff --git a/hadoop-tools/hadoop-federation-balance/pom.xml b/hadoop-tools/hadoop-federation-balance/pom.xml index 588bb98f3e75ad..71f2cb3639137e 100644 --- a/hadoop-tools/hadoop-federation-balance/pom.xml +++ b/hadoop-tools/hadoop-federation-balance/pom.xml @@ -138,7 +138,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} 1 false 600 From 056450e71d60de479614ec7b3313d9cbecb7bb7c Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Thu, 9 Dec 2021 17:51:44 +0100 Subject: [PATCH 03/33] YARN-10982. Replace all occurences of queuePath with the new QueuePath class. Contributed by Tibor Kovacs --- .../scheduler/capacity/AbstractCSQueue.java | 25 +++--- .../capacity/AbstractManagedParentQueue.java | 2 +- .../capacity/AutoCreatedQueueTemplate.java | 14 ++- .../scheduler/capacity/CSQueue.java | 6 ++ .../scheduler/capacity/CSQueueUtils.java | 8 +- .../CapacitySchedulerConfiguration.java | 57 ++++++------ .../capacity/ManagedParentQueue.java | 9 +- .../scheduler/capacity/ParentQueue.java | 22 ++--- .../scheduler/capacity/QueuePath.java | 23 +++++ .../TestAbsoluteResourceConfiguration.java | 90 ++++++++++--------- .../TestAbsoluteResourceWithAutoQueue.java | 18 ++-- .../TestAutoCreatedQueueTemplate.java | 57 ++++++------ .../TestCSAllocateCustomResource.java | 4 +- .../capacity/TestCapacityScheduler.java | 9 +- .../TestCapacitySchedulerConfigValidator.java | 35 ++++---- .../scheduler/capacity/TestParentQueue.java | 4 +- .../scheduler/capacity/TestQueuePath.java | 10 +++ ...estRMWebServicesConfigurationMutation.java | 15 ++-- 18 files changed, 233 insertions(+), 175 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index efdfa8e5af8806..097a9dfbc57763 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -82,7 +82,6 @@ public abstract class AbstractCSQueue implements CSQueue { protected final QueueAllocationSettings queueAllocationSettings; volatile CSQueue parent; protected final QueuePath queuePath; - final String queueName; protected QueueNodeLabelsSettings queueNodeLabelsSettings; private volatile QueueAppLifetimeAndLimitSettings queueAppLifetimeSettings; private CSQueuePreemptionSettings preemptionSettings; @@ -143,7 +142,6 @@ public AbstractCSQueue(CapacitySchedulerContext cs, this.labelManager = cs.getRMContext().getNodeLabelManager(); this.parent = parent; this.queuePath = createQueuePath(parent, queueName); - this.queueName = queuePath.getLeafName(); this.resourceCalculator = cs.getResourceCalculator(); this.activitiesManager = cs.getActivitiesManager(); @@ -176,7 +174,7 @@ protected void setupConfigurableCapacities() { protected void setupConfigurableCapacities( CapacitySchedulerConfiguration configuration) { - CSQueueUtils.loadCapacitiesByLabelsFromConf(getQueuePath(), queueCapacities, + CSQueueUtils.loadCapacitiesByLabelsFromConf(queuePath, queueCapacities, configuration, this.queueNodeLabelsSettings.getConfiguredNodeLabels()); } @@ -185,6 +183,11 @@ public String getQueuePath() { return queuePath.getFullPath(); } + @Override + public QueuePath getQueuePathObject() { + return this.queuePath; + } + @Override public float getCapacity() { return queueCapacities.getCapacity(); @@ -241,7 +244,7 @@ public String getQueueShortName() { @Override public String getQueueName() { - return queueName; + return this.queuePath.getLeafName(); } @Override @@ -279,11 +282,11 @@ void setMaxCapacity(float maximumCapacity) { writeLock.lock(); try { // Sanity check - CSQueueUtils.checkMaxCapacity(getQueuePath(), + CSQueueUtils.checkMaxCapacity(this.queuePath, queueCapacities.getCapacity(), maximumCapacity); float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity( maximumCapacity, parent); - CSQueueUtils.checkAbsoluteCapacity(getQueuePath(), + CSQueueUtils.checkAbsoluteCapacity(this.queuePath, queueCapacities.getAbsoluteCapacity(), absMaxCapacity); queueCapacities.setMaximumCapacity(maximumCapacity); @@ -301,11 +304,11 @@ void setMaxCapacity(String nodeLabel, float maximumCapacity) { writeLock.lock(); try { // Sanity check - CSQueueUtils.checkMaxCapacity(getQueuePath(), + CSQueueUtils.checkMaxCapacity(this.queuePath, queueCapacities.getCapacity(nodeLabel), maximumCapacity); float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity( maximumCapacity, parent); - CSQueueUtils.checkAbsoluteCapacity(getQueuePath(), + CSQueueUtils.checkAbsoluteCapacity(this.queuePath, queueCapacities.getAbsoluteCapacity(nodeLabel), absMaxCapacity); queueCapacities.setMaximumCapacity(maximumCapacity); @@ -518,7 +521,7 @@ private void validateMinResourceIsNotGreaterThanMaxResource(Resource minResource private void validateAbsoluteVsPercentageCapacityConfig( CapacityConfigType localType) { - if (!getQueuePath().equals("root") + if (!queuePath.isRoot() && !this.capacityConfigType.equals(localType)) { throw new IllegalArgumentException("Queue '" + getQueuePath() + "' should use either percentage based capacity" @@ -623,8 +626,8 @@ protected QueueInfo getQueueInfo() { // consistency here. // TODO, improve this QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); - queueInfo.setQueueName(queueName); - queueInfo.setQueuePath(getQueuePath()); + queueInfo.setQueueName(queuePath.getLeafName()); + queueInfo.setQueuePath(queuePath.getFullPath()); queueInfo.setAccessibleNodeLabels(queueNodeLabelsSettings.getAccessibleNodeLabels()); queueInfo.setCapacity(queueCapacities.getCapacity()); queueInfo.setMaximumCapacity(queueCapacities.getMaximumCapacity()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java index 9c16de097d6aa2..7d149761cb0dba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java @@ -208,7 +208,7 @@ protected void validateQueueEntitlementChange(AbstractAutoCreatedLeafQueue if (!(newChildCap >= 0 && newChildCap < 1.0f + CSQueueUtils.EPSILON)) { throw new SchedulerDynamicEditException( "Sum of child queues should exceed 100% for auto creating parent " - + "queue : " + queueName); + + "queue : " + getQueueName()); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java index 0a3d49a50322b9..1603b19cf2eaf8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java @@ -20,13 +20,12 @@ import org.apache.hadoop.classification.VisibleForTesting; -import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.hadoop.util.Lists; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_QUEUE_CREATION_V2_PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix; @@ -50,7 +49,7 @@ public class AutoCreatedQueueTemplate { private final Map parentOnlyProperties = new HashMap<>(); public AutoCreatedQueueTemplate(CapacitySchedulerConfiguration configuration, - String queuePath) { + QueuePath queuePath) { setTemplateConfigEntries(configuration, queuePath); } @@ -155,14 +154,13 @@ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, * yarn.scheduler.capacity.root.*.auto-queue-creation-v2.template.capacity */ private void setTemplateConfigEntries(CapacitySchedulerConfiguration configuration, - String queuePath) { + QueuePath queuePath) { ConfigurationProperties configurationProperties = configuration.getConfigurationProperties(); - List queuePathParts = new ArrayList<>(Arrays.asList( - queuePath.split("\\."))); + List queuePathParts = Lists.newArrayList(queuePath.iterator()); - if (queuePathParts.size() <= 1 && !queuePath.equals(ROOT)) { + if (queuePathParts.size() <= 1 && !queuePath.isRoot()) { // This is an invalid queue path return; } @@ -175,7 +173,7 @@ private void setTemplateConfigEntries(CapacitySchedulerConfiguration configurati int supportedWildcardLevel = Math.min(queuePathMaxIndex - 1, MAX_WILDCARD_LEVEL); // Allow root to have template properties - if (queuePath.equals(ROOT)) { + if (queuePath.isRoot()) { supportedWildcardLevel = 0; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index 2acc1d4b9baf23..90cb4f34ddeccd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -89,6 +89,12 @@ public interface CSQueue extends SchedulerQueue { */ public String getQueuePath(); + /** + * Gets the queue path object. + * @return the object of the queue + */ + QueuePath getQueuePathObject(); + public PrivilegedEntity getPrivilegedEntity(); Resource getMaximumAllocation(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 410117a9127023..244bb62d508d82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -36,7 +36,7 @@ public class CSQueueUtils { /* * Used only by tests */ - public static void checkMaxCapacity(String queuePath, + public static void checkMaxCapacity(QueuePath queuePath, float capacity, float maximumCapacity) { if (maximumCapacity < 0.0f || maximumCapacity > 1.0f) { throw new IllegalArgumentException( @@ -48,7 +48,7 @@ public static void checkMaxCapacity(String queuePath, /* * Used only by tests */ - public static void checkAbsoluteCapacity(String queuePath, + public static void checkAbsoluteCapacity(QueuePath queuePath, float absCapacity, float absMaxCapacity) { if (absMaxCapacity < (absCapacity - EPSILON)) { throw new IllegalArgumentException("Illegal call to setMaxCapacity. " @@ -67,7 +67,7 @@ public static float computeAbsoluteMaximumCapacity( } public static void loadCapacitiesByLabelsFromConf( - String queuePath, QueueCapacities queueCapacities, + QueuePath queuePath, QueueCapacities queueCapacities, CapacitySchedulerConfiguration csConf, Set nodeLabels) { queueCapacities.clearConfigurableFields(); @@ -81,7 +81,7 @@ public static void loadCapacitiesByLabelsFromConf( label, csConf.getMaximumAMResourcePercentPerPartition(queuePath, label)); queueCapacities.setWeight(label, - csConf.getNonLabeledQueueWeight(queuePath)); + csConf.getNonLabeledQueueWeight(queuePath.getFullPath())); } else{ queueCapacities.setCapacity(label, csConf.getLabeledQueueCapacity(queuePath, label) / 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 615a4d08087d45..e88f83a44cbc2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -542,15 +542,15 @@ public void setLabeledQueueWeight(String queue, String label, float weight) { set(getNodeLabelPrefix(queue, label) + CAPACITY, weight + WEIGHT_SUFFIX); } - public float getLabeledQueueWeight(String queue, String label) { - String configuredValue = get(getNodeLabelPrefix(queue, label) + CAPACITY); + public float getLabeledQueueWeight(QueuePath queue, String label) { + String configuredValue = get(getNodeLabelPrefix(queue.getFullPath(), label) + CAPACITY); float weight = extractFloatValueFromWeightConfig(configuredValue); - throwExceptionForUnexpectedWeight(weight, queue, label); + throwExceptionForUnexpectedWeight(weight, queue.getFullPath(), label); return weight; } - public float getNonLabeledQueueCapacity(String queue) { - String configuredCapacity = get(getQueuePrefix(queue) + CAPACITY); + public float getNonLabeledQueueCapacity(QueuePath queue) { + String configuredCapacity = get(getQueuePrefix(queue.getFullPath()) + CAPACITY); boolean absoluteResourceConfigured = (configuredCapacity != null) && RESOURCE_PATTERN.matcher(configuredCapacity).find(); if (absoluteResourceConfigured || configuredWeightAsCapacity( @@ -559,10 +559,10 @@ public float getNonLabeledQueueCapacity(String queue) { // root.From AbstractCSQueue, absolute resource will be parsed and // updated. Once nodes are added/removed in cluster, capacity in // percentage will also be re-calculated. - return queue.equals("root") ? 100.0f : 0f; + return queue.isRoot() ? 100.0f : 0f; } - float capacity = queue.equals("root") + float capacity = queue.isRoot() ? 100.0f : (configuredCapacity == null) ? 0f @@ -573,7 +573,7 @@ public float getNonLabeledQueueCapacity(String queue) { "Illegal " + "capacity of " + capacity + " for queue " + queue); } LOG.debug("CSConf - getCapacity: queuePrefix={}, capacity={}", - getQueuePrefix(queue), capacity); + getQueuePrefix(queue.getFullPath()), capacity); return capacity; } @@ -601,8 +601,8 @@ public void setCapacity(String queue, String absoluteResourceCapacity) { } - public float getNonLabeledQueueMaximumCapacity(String queue) { - String configuredCapacity = get(getQueuePrefix(queue) + MAXIMUM_CAPACITY); + public float getNonLabeledQueueMaximumCapacity(QueuePath queue) { + String configuredCapacity = get(getQueuePrefix(queue.getFullPath()) + MAXIMUM_CAPACITY); boolean matcher = (configuredCapacity != null) && RESOURCE_PATTERN.matcher(configuredCapacity).find(); if (matcher) { @@ -816,9 +816,9 @@ private float extractFloatValueFromWeightConfig(String configureValue) { } } - private float internalGetLabeledQueueCapacity(String queue, String label, + private float internalGetLabeledQueueCapacity(QueuePath queue, String label, String suffix, float defaultValue) { - String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix; + String capacityPropertyName = getNodeLabelPrefix(queue.getFullPath(), label) + suffix; String configuredCapacity = get(capacityPropertyName); boolean absoluteResourceConfigured = (configuredCapacity != null) && RESOURCE_PATTERN.matcher( @@ -829,10 +829,10 @@ private float internalGetLabeledQueueCapacity(String queue, String label, // root.From AbstractCSQueue, absolute resource, and weight will be parsed // and updated separately. Once nodes are added/removed in cluster, // capacity is percentage will also be re-calculated. - return queue.equals("root") ? 100.0f : defaultValue; + return queue.isRoot() ? 100.0f : defaultValue; } - float capacity = queue.equals("root") ? 100.0f + float capacity = queue.isRoot() ? 100.0f : getFloat(capacityPropertyName, defaultValue); if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) { @@ -843,17 +843,17 @@ private float internalGetLabeledQueueCapacity(String queue, String label, } if (LOG.isDebugEnabled()) { LOG.debug( - "CSConf - getCapacityOfLabel: prefix=" + getNodeLabelPrefix(queue, + "CSConf - getCapacityOfLabel: prefix=" + getNodeLabelPrefix(queue.getFullPath(), label) + ", capacity=" + capacity); } return capacity; } - public float getLabeledQueueCapacity(String queue, String label) { + public float getLabeledQueueCapacity(QueuePath queue, String label) { return internalGetLabeledQueueCapacity(queue, label, CAPACITY, 0f); } - public float getLabeledQueueMaximumCapacity(String queue, String label) { + public float getLabeledQueueMaximumCapacity(QueuePath queue, String label) { return internalGetLabeledQueueCapacity(queue, label, MAXIMUM_CAPACITY, 100f); } @@ -870,13 +870,13 @@ public void setDefaultNodeLabelExpression(String queue, String exp) { set(getQueuePrefix(queue) + DEFAULT_NODE_LABEL_EXPRESSION, exp); } - public float getMaximumAMResourcePercentPerPartition(String queue, + public float getMaximumAMResourcePercentPerPartition(QueuePath queue, String label) { // If per-partition max-am-resource-percent is not configured, // use default value as max-am-resource-percent for this queue. - return getFloat(getNodeLabelPrefix(queue, label) + return getFloat(getNodeLabelPrefix(queue.getFullPath(), label) + MAXIMUM_AM_RESOURCE_SUFFIX, - getMaximumApplicationMasterResourcePerQueuePercent(queue)); + getMaximumApplicationMasterResourcePerQueuePercent(queue.getFullPath())); } public void setMaximumAMResourcePercentPerPartition(String queue, @@ -2189,6 +2189,11 @@ public String getAutoCreatedQueueTemplateConfPrefix(String queuePath) { return queuePath + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX; } + @Private + public QueuePath getAutoCreatedQueueObjectTemplateConfPrefix(String queuePath) { + return new QueuePath(queuePath, AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX); + } + @Private public static final String FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY = "auto-create-child-queue.fail-on-exceeding-parent-capacity"; @@ -2565,13 +2570,13 @@ public Resource getMaximumResourceRequirement(String label, String queue, } @VisibleForTesting - public void setMinimumResourceRequirement(String label, String queue, + public void setMinimumResourceRequirement(String label, QueuePath queue, Resource resource) { updateMinMaxResourceToConf(label, queue, resource, CAPACITY); } @VisibleForTesting - public void setMaximumResourceRequirement(String label, String queue, + public void setMaximumResourceRequirement(String label, QueuePath queue, Resource resource) { updateMinMaxResourceToConf(label, queue, resource, MAXIMUM_CAPACITY); } @@ -2586,9 +2591,9 @@ public Map parseConfiguredResourceVector( return queueResourceVectors; } - private void updateMinMaxResourceToConf(String label, String queue, + private void updateMinMaxResourceToConf(String label, QueuePath queue, Resource resource, String type) { - if (queue.equals("root")) { + if (queue.isRoot()) { throw new IllegalArgumentException( "Cannot set resource, root queue will take 100% of cluster capacity"); } @@ -2603,9 +2608,9 @@ private void updateMinMaxResourceToConf(String label, String queue, + ResourceUtils. getCustomResourcesStrings(resource) + "]"); - String prefix = getQueuePrefix(queue) + type; + String prefix = getQueuePrefix(queue.getFullPath()) + type; if (!label.isEmpty()) { - prefix = getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label + prefix = getQueuePrefix(queue.getFullPath()) + ACCESSIBLE_NODE_LABELS + DOT + label + DOT + type; } set(prefix, resourceString.toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index e415ac12795853..6e7325c3a9747e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -23,7 +23,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler .SchedulerDynamicEditException; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue.CapacityConfigType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.queuemanagement.GuaranteedOrZeroCapacityOverTimePolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica .FiCaSchedulerApp; @@ -122,7 +121,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) LOG.info( "Reinitialized Managed Parent Queue: [{}] with capacity [{}]" + " with max capacity [{}]", - queueName, super.getCapacity(), super.getMaximumCapacity()); + getQueueName(), super.getCapacity(), super.getMaximumCapacity()); } catch (YarnException ye) { LOG.error("Exception while computing policy changes for leaf queue : " + getQueuePath(), ye); @@ -165,12 +164,12 @@ protected AutoCreatedLeafQueueConfig.Builder initializeLeafQueueConfigs() throws CapacitySchedulerConfiguration conf = super.initializeLeafQueueConfigs(leafQueueTemplateConfPrefix); builder.configuration(conf); - String templateQueuePath = csContext.getConfiguration() - .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()); + QueuePath templateQueuePath = csContext.getConfiguration() + .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePath()); Set templateConfiguredNodeLabels = csContext .getCapacitySchedulerQueueManager().getConfiguredNodeLabels() - .getLabelsByQueue(templateQueuePath); + .getLabelsByQueue(templateQueuePath.getFullPath()); for (String nodeLabel : templateConfiguredNodeLabels) { Resource templateMinResource = conf.getMinimumResourceRequirement( nodeLabel, csContext.getConfiguration() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 0f302b8e73c990..aec2bd8468db55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -128,7 +128,7 @@ private ParentQueue(CapacitySchedulerContext cs, this.scheduler = cs; this.rootQueue = (parent == null); - float rawCapacity = csConf.getNonLabeledQueueCapacity(getQueuePath()); + float rawCapacity = csConf.getNonLabeledQueueCapacity(this.queuePath); if (rootQueue && (rawCapacity != CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE)) { @@ -161,7 +161,7 @@ protected void setupQueueConfigs(Resource clusterResource, writeLock.lock(); try { autoCreatedQueueTemplate = new AutoCreatedQueueTemplate( - csConf, getQueuePath()); + csConf, this.queuePath); super.setupQueueConfigs(clusterResource, csConf); StringBuilder aclsString = new StringBuilder(); for (Map.Entry e : acls.entrySet()) { @@ -182,7 +182,7 @@ protected void setupQueueConfigs(Resource clusterResource, ((ParentQueue) parent).getQueueOrderingPolicyConfigName()); queueOrderingPolicy.setQueues(childQueues); - LOG.info(queueName + ", " + getCapacityOrWeightString() + LOG.info(getQueueName() + ", " + getCapacityOrWeightString() + ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() + ", maxCapacity=" + this.queueCapacities.getMaximumCapacity() + ", absoluteMaxCapacity=" + this.queueCapacities @@ -333,7 +333,7 @@ void setChildQueues(Collection childQueues) throws IOException { throw new IOException( "Parent Queues" + " capacity: " + parentMinResource + " is less than" + " to its children:" + minRes - + " for queue:" + queueName); + + " for queue:" + getQueueName()); } } } @@ -355,7 +355,7 @@ void setChildQueues(Collection childQueues) throws IOException { // It is wrong when percent sum != {0, 1} throw new IOException( "Illegal" + " capacity sum of " + childrenPctSum - + " for children of queue " + queueName + " for label=" + + " for children of queue " + getQueueName() + " for label=" + nodeLabel + ". It should be either 0 or 1.0"); } else{ // We also allow children's percent sum = 0 under the following @@ -368,7 +368,7 @@ void setChildQueues(Collection childQueues) throws IOException { > PRECISION) && (!allowZeroCapacitySum)) { throw new IOException( "Illegal" + " capacity sum of " + childrenPctSum - + " for children of queue " + queueName + + " for children of queue " + getQueueName() + " for label=" + nodeLabel + ". It is set to 0, but parent percent != 0, and " + "doesn't allow children capacity to set to 0"); @@ -383,8 +383,8 @@ void setChildQueues(Collection childQueues) throws IOException { && !allowZeroCapacitySum) { throw new IOException( "Illegal" + " capacity sum of " + childrenPctSum - + " for children of queue " + queueName + " for label=" - + nodeLabel + ". queue=" + queueName + + " for children of queue " + getQueueName() + " for label=" + + nodeLabel + ". queue=" + getQueueName() + " has zero capacity, but child" + "queues have positive capacities"); } @@ -470,7 +470,7 @@ public List getQueueUserAclInfo( } public String toString() { - return queueName + ": " + + return getQueueName() + ": " + "numChildQueue= " + childQueues.size() + ", " + getCapacityOrWeightString() + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " + @@ -768,9 +768,9 @@ public void validateSubmitApplication(ApplicationId applicationId, String userName, String queue) throws AccessControlException { writeLock.lock(); try { - if (queue.equals(queueName)) { + if (queue.equals(getQueueName())) { throw new AccessControlException( - "Cannot submit application " + "to non-leaf queue: " + queueName); + "Cannot submit application " + "to non-leaf queue: " + getQueueName()); } if (getState() != QueueState.RUNNING) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java index 3ca77359185760..37cfa2ef73366d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java @@ -24,6 +24,7 @@ import java.util.Objects; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; /** * This is a helper class which represents a queue path, and has easy access @@ -59,6 +60,15 @@ public QueuePath(String fullPath) { setFromFullPath(fullPath); } + /** + * Concatenate queue path parts into one queue path string. + * @param parts Parts of the full queue pathAutoCreatedQueueTemplate + * @return full path of the given queue parts + */ + public static String concatenatePath(String... parts) { + return String.join(DOT, parts); + } + /** * This method is responsible for splitting up a full queue path into parent * path and leaf name. @@ -68,6 +78,11 @@ private void setFromFullPath(String fullPath) { parent = null; leaf = fullPath; + if (leaf == null) { + leaf = ""; + return; + } + int lastDotIdx = fullPath.lastIndexOf(DOT); if (lastDotIdx > -1) { parent = fullPath.substring(0, lastDotIdx).trim(); @@ -121,6 +136,14 @@ public boolean hasParent() { return parent != null; } + /** + * Convenience getter to check if the queue is the root queue. + * @return True if the path is root + */ + public boolean isRoot() { + return !hasParent() && leaf.equals(ROOT); + } + /** * Creates a new {@code QueuePath} from the current full path as parent, and * the appended child queue path as leaf. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index 7114728fc6b7af..08462332818cfa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -44,18 +44,21 @@ public class TestAbsoluteResourceConfiguration { private static final String QUEUEA2 = "queueA2"; private static final String QUEUEB1 = "queueB1"; - private static final String QUEUEA_FULL = CapacitySchedulerConfiguration.ROOT - + "." + QUEUEA; - private static final String QUEUEB_FULL = CapacitySchedulerConfiguration.ROOT - + "." + QUEUEB; - private static final String QUEUEC_FULL = CapacitySchedulerConfiguration.ROOT - + "." + QUEUEC; - private static final String QUEUED_FULL = CapacitySchedulerConfiguration.ROOT - + "." + QUEUED; - - private static final String QUEUEA1_FULL = QUEUEA_FULL + "." + QUEUEA1; - private static final String QUEUEA2_FULL = QUEUEA_FULL + "." + QUEUEA2; - private static final String QUEUEB1_FULL = QUEUEB_FULL + "." + QUEUEB1; + private static final QueuePath QUEUEA_FULL = + new QueuePath(CapacitySchedulerConfiguration.ROOT, QUEUEA); + private static final QueuePath QUEUEB_FULL = + new QueuePath(CapacitySchedulerConfiguration.ROOT, QUEUEB); + private static final QueuePath QUEUEC_FULL = + new QueuePath(CapacitySchedulerConfiguration.ROOT, QUEUEC); + private static final QueuePath QUEUED_FULL = + new QueuePath(CapacitySchedulerConfiguration.ROOT, QUEUED); + + private static final QueuePath QUEUEA1_FULL = + new QueuePath(QUEUEA_FULL.getFullPath() + "." + QUEUEA1); + private static final QueuePath QUEUEA2_FULL = + new QueuePath(QUEUEA_FULL.getFullPath() + "." + QUEUEA2); + private static final QueuePath QUEUEB1_FULL = + new QueuePath(QUEUEB_FULL.getFullPath() + "." + QUEUEB1); private static final Resource QUEUE_A_MINRES = Resource.newInstance(100 * GB, 10); @@ -100,18 +103,18 @@ private CapacitySchedulerConfiguration setupSimpleQueueConfiguration( // Set default capacities like normal configuration. if (isCapacityNeeded) { - csConf.setCapacity(QUEUEA_FULL, 50f); - csConf.setCapacity(QUEUEB_FULL, 25f); - csConf.setCapacity(QUEUEC_FULL, 25f); - csConf.setCapacity(QUEUED_FULL, 25f); + csConf.setCapacity(QUEUEA_FULL.getFullPath(), 50f); + csConf.setCapacity(QUEUEB_FULL.getFullPath(), 25f); + csConf.setCapacity(QUEUEC_FULL.getFullPath(), 25f); + csConf.setCapacity(QUEUED_FULL.getFullPath(), 25f); } - csConf.setAutoCreateChildQueueEnabled(QUEUED_FULL, true); + csConf.setAutoCreateChildQueueEnabled(QUEUED_FULL.getFullPath(), true); // Setup leaf queue template configs - csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(QUEUED_FULL, "", + csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(QUEUED_FULL.getFullPath(), "", QUEUE_D_TEMPL_MINRES); - csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(QUEUED_FULL, "", + csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(QUEUED_FULL.getFullPath(), "", QUEUE_D_TEMPL_MAXRES); return csConf; @@ -122,17 +125,17 @@ private CapacitySchedulerConfiguration setupComplexQueueConfiguration( CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{QUEUEA, QUEUEB, QUEUEC}); - csConf.setQueues(QUEUEA_FULL, new String[]{QUEUEA1, QUEUEA2}); - csConf.setQueues(QUEUEB_FULL, new String[]{QUEUEB1}); + csConf.setQueues(QUEUEA_FULL.getFullPath(), new String[]{QUEUEA1, QUEUEA2}); + csConf.setQueues(QUEUEB_FULL.getFullPath(), new String[]{QUEUEB1}); // Set default capacities like normal configuration. if (isCapacityNeeded) { - csConf.setCapacity(QUEUEA_FULL, 50f); - csConf.setCapacity(QUEUEB_FULL, 25f); - csConf.setCapacity(QUEUEC_FULL, 25f); - csConf.setCapacity(QUEUEA1_FULL, 50f); - csConf.setCapacity(QUEUEA2_FULL, 50f); - csConf.setCapacity(QUEUEB1_FULL, 100f); + csConf.setCapacity(QUEUEA_FULL.getFullPath(), 50f); + csConf.setCapacity(QUEUEB_FULL.getFullPath(), 25f); + csConf.setCapacity(QUEUEC_FULL.getFullPath(), 25f); + csConf.setCapacity(QUEUEA1_FULL.getFullPath(), 50f); + csConf.setCapacity(QUEUEA2_FULL.getFullPath(), 50f); + csConf.setCapacity(QUEUEB1_FULL.getFullPath(), 100f); } return csConf; @@ -140,6 +143,7 @@ private CapacitySchedulerConfiguration setupComplexQueueConfiguration( private CapacitySchedulerConfiguration setupMinMaxResourceConfiguration( CapacitySchedulerConfiguration csConf) { + // Update min/max resource to queueA/B/C csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES); csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES); @@ -180,22 +184,22 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue() Assert.assertEquals("Min resource configured for QUEUEA is not correct", QUEUE_A_MINRES, - csConf.getMinimumResourceRequirement("", QUEUEA_FULL, resourceTypes)); + csConf.getMinimumResourceRequirement("", QUEUEA_FULL.getFullPath(), resourceTypes)); Assert.assertEquals("Max resource configured for QUEUEA is not correct", QUEUE_A_MAXRES, - csConf.getMaximumResourceRequirement("", QUEUEA_FULL, resourceTypes)); + csConf.getMaximumResourceRequirement("", QUEUEA_FULL.getFullPath(), resourceTypes)); Assert.assertEquals("Min resource configured for QUEUEB is not correct", QUEUE_B_MINRES, - csConf.getMinimumResourceRequirement("", QUEUEB_FULL, resourceTypes)); + csConf.getMinimumResourceRequirement("", QUEUEB_FULL.getFullPath(), resourceTypes)); Assert.assertEquals("Max resource configured for QUEUEB is not correct", QUEUE_B_MAXRES, - csConf.getMaximumResourceRequirement("", QUEUEB_FULL, resourceTypes)); + csConf.getMaximumResourceRequirement("", QUEUEB_FULL.getFullPath(), resourceTypes)); Assert.assertEquals("Min resource configured for QUEUEC is not correct", QUEUE_C_MINRES, - csConf.getMinimumResourceRequirement("", QUEUEC_FULL, resourceTypes)); + csConf.getMinimumResourceRequirement("", QUEUEC_FULL.getFullPath(), resourceTypes)); Assert.assertEquals("Max resource configured for QUEUEC is not correct", QUEUE_C_MAXRES, - csConf.getMaximumResourceRequirement("", QUEUEC_FULL, resourceTypes)); + csConf.getMaximumResourceRequirement("", QUEUEC_FULL.getFullPath(), resourceTypes)); csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); @@ -487,9 +491,9 @@ public void testComplexValidateAbsoluteResourceConfig() throws Exception { // 1. Explicitly set percentage based config for parent queues. This will // make Queue A,B and C with percentage based and A1,A2 or B1 with absolute // resource. - csConf.setCapacity(QUEUEA_FULL, 50f); - csConf.setCapacity(QUEUEB_FULL, 25f); - csConf.setCapacity(QUEUEC_FULL, 25f); + csConf.setCapacity(QUEUEA_FULL.getFullPath(), 50f); + csConf.setCapacity(QUEUEB_FULL.getFullPath(), 25f); + csConf.setCapacity(QUEUEC_FULL.getFullPath(), 25f); // Get queue object to verify min/max resource configuration. CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); @@ -534,13 +538,13 @@ public void testValidateAbsoluteResourceConfig() throws Exception { new CapacitySchedulerConfiguration(); csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {QUEUEA, QUEUEB}); - csConf.setQueues(QUEUEA_FULL, new String[] {QUEUEA1, QUEUEA2}); + csConf.setQueues(QUEUEA_FULL.getFullPath(), new String[] {QUEUEA1, QUEUEA2}); // Set default capacities like normal configuration. - csConf.setCapacity(QUEUEA_FULL, "[memory=125]"); - csConf.setCapacity(QUEUEB_FULL, "[memory=0]"); - csConf.setCapacity(QUEUEA1_FULL, "[memory=100]"); - csConf.setCapacity(QUEUEA2_FULL, "[memory=25]"); + csConf.setCapacity(QUEUEA_FULL.getFullPath(), "[memory=125]"); + csConf.setCapacity(QUEUEB_FULL.getFullPath(), "[memory=0]"); + csConf.setCapacity(QUEUEA1_FULL.getFullPath(), "[memory=100]"); + csConf.setCapacity(QUEUEA2_FULL.getFullPath(), "[memory=25]"); // Update min/max resource to queueA csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES); @@ -560,8 +564,8 @@ public void testValidateAbsoluteResourceConfig() throws Exception { // doesnt throw exception saying "Parent queue 'root.A' and // child queue 'root.A.A2' should use either percentage // based capacityconfiguration or absolute resource together for label" - csConf.setCapacity(QUEUEA1_FULL, "[memory=125]"); - csConf.setCapacity(QUEUEA2_FULL, "[memory=0]"); + csConf.setCapacity(QUEUEA1_FULL.getFullPath(), "[memory=125]"); + csConf.setCapacity(QUEUEA2_FULL.getFullPath(), "[memory=0]"); // Get queue object to verify min/max resource configuration. CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java index 2bad8b7447b4f1..326b9d0795c7ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java @@ -96,15 +96,15 @@ public void setUp() throws Exception { private CapacitySchedulerConfiguration setupMinMaxResourceConfiguration( CapacitySchedulerConfiguration csConf) { // Update min/max resource to queueA/B/C - csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES); - csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES); - csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES); - csConf.setMinimumResourceRequirement("", QUEUED_FULL, QUEUE_D_MINRES); - - csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES); - csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES); - csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES); - csConf.setMaximumResourceRequirement("", QUEUED_FULL, QUEUE_D_MAXRES); + csConf.setMinimumResourceRequirement("", new QueuePath(QUEUEA_FULL), QUEUE_A_MINRES); + csConf.setMinimumResourceRequirement("", new QueuePath(QUEUEB_FULL), QUEUE_B_MINRES); + csConf.setMinimumResourceRequirement("", new QueuePath(QUEUEC_FULL), QUEUE_C_MINRES); + csConf.setMinimumResourceRequirement("", new QueuePath(QUEUED_FULL), QUEUE_D_MINRES); + + csConf.setMaximumResourceRequirement("", new QueuePath(QUEUEA_FULL), QUEUE_A_MAXRES); + csConf.setMaximumResourceRequirement("", new QueuePath(QUEUEB_FULL), QUEUE_B_MAXRES); + csConf.setMaximumResourceRequirement("", new QueuePath(QUEUEC_FULL), QUEUE_C_MAXRES); + csConf.setMaximumResourceRequirement("", new QueuePath(QUEUED_FULL), QUEUE_D_MAXRES); return csConf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java index 37f1378d7c6413..5b58feb21faa40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java @@ -25,11 +25,12 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE; public class TestAutoCreatedQueueTemplate { - private static final String TEST_QUEUE_ABC = "root.a.b.c"; - private static final String TEST_QUEUE_AB = "root.a.b"; - private static final String TEST_QUEUE_A = "root.a"; - private static final String TEST_QUEUE_B = "root.b"; + private static final QueuePath TEST_QUEUE_ABC = new QueuePath("root.a.b.c"); + private static final QueuePath TEST_QUEUE_AB = new QueuePath("root.a.b"); + private static final QueuePath TEST_QUEUE_A = new QueuePath("root.a"); + private static final QueuePath TEST_QUEUE_B = new QueuePath("root.b"); private static final String ROOT = "root"; + private CapacitySchedulerConfiguration conf; @Before @@ -43,13 +44,13 @@ public void setUp() throws Exception { @Test public void testNonWildCardTemplate() { - conf.set(getTemplateKey(TEST_QUEUE_AB, "capacity"), "6w"); + conf.set(getTemplateKey(TEST_QUEUE_AB.getFullPath(), "capacity"), "6w"); AutoCreatedQueueTemplate template = new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath()); Assert.assertEquals("weight is not set", 6f, - conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); + conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6); } @@ -58,10 +59,10 @@ public void testOneLevelWildcardTemplate() { conf.set(getTemplateKey("root.a.*", "capacity"), "6w"); AutoCreatedQueueTemplate template = new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath()); Assert.assertEquals("weight is not set", 6f, - conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); + conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6); } @@ -69,18 +70,18 @@ public void testOneLevelWildcardTemplate() { public void testIgnoredWhenRootWildcarded() { conf.set(getTemplateKey("*", "capacity"), "6w"); AutoCreatedQueueTemplate template = - new AutoCreatedQueueTemplate(conf, ROOT); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_A); + new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT)); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_A.getFullPath()); Assert.assertEquals("weight is set", -1f, - conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6); + conf.getNonLabeledQueueWeight(TEST_QUEUE_A.getFullPath()), 10e-6); } @Test public void testIgnoredWhenNoParent() { conf.set(getTemplateKey("root", "capacity"), "6w"); AutoCreatedQueueTemplate template = - new AutoCreatedQueueTemplate(conf, ROOT); + new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT)); template.setTemplateEntriesForChild(conf, ROOT); Assert.assertEquals("weight is set", -1f, @@ -95,21 +96,21 @@ public void testTemplatePrecedence() { AutoCreatedQueueTemplate template = new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath()); Assert.assertEquals( "explicit template does not have the highest precedence", 6f, - conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); + conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6); CapacitySchedulerConfiguration newConf = new CapacitySchedulerConfiguration(); newConf.set(getTemplateKey("root.a.*", "capacity"), "4w"); template = new AutoCreatedQueueTemplate(newConf, TEST_QUEUE_AB); - template.setTemplateEntriesForChild(newConf, TEST_QUEUE_ABC); + template.setTemplateEntriesForChild(newConf, TEST_QUEUE_ABC.getFullPath()); Assert.assertEquals("precedence is invalid", 4f, - newConf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); + newConf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6); } @Test @@ -117,10 +118,10 @@ public void testRootTemplate() { conf.set(getTemplateKey("root", "capacity"), "2w"); AutoCreatedQueueTemplate template = - new AutoCreatedQueueTemplate(conf, ROOT); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_A); + new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT)); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_A.getFullPath()); Assert.assertEquals("root property is not set", 2f, - conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6); + conf.getNonLabeledQueueWeight(TEST_QUEUE_A.getFullPath()), 10e-6); } @Test @@ -133,21 +134,21 @@ public void testQueueSpecificTemplates() { "root", AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE), false); AutoCreatedQueueTemplate template = - new AutoCreatedQueueTemplate(conf, ROOT); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_A); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_B, true); + new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT)); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_A.getFullPath()); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_B.getFullPath(), true); Assert.assertNull("default-node-label-expression is set for parent", - conf.getDefaultNodeLabelExpression(TEST_QUEUE_A)); + conf.getDefaultNodeLabelExpression(TEST_QUEUE_A.getFullPath())); Assert.assertEquals("default-node-label-expression is not set for leaf", - "test", conf.getDefaultNodeLabelExpression(TEST_QUEUE_B)); + "test", conf.getDefaultNodeLabelExpression(TEST_QUEUE_B.getFullPath())); Assert.assertFalse("auto queue removal is not disabled for parent", - conf.isAutoExpiredDeletionEnabled(TEST_QUEUE_A)); + conf.isAutoExpiredDeletionEnabled(TEST_QUEUE_A.getFullPath())); Assert.assertEquals("weight should not be overridden when set by " + "queue type specific template", - 10f, conf.getNonLabeledQueueWeight(TEST_QUEUE_B), 10e-6); + 10f, conf.getNonLabeledQueueWeight(TEST_QUEUE_B.getFullPath()), 10e-6); Assert.assertEquals("weight should be set by common template", - 2f, conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6); + 2f, conf.getNonLabeledQueueWeight(TEST_QUEUE_A.getFullPath()), 10e-6); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java index 36b3c9b4d63ffc..5b233f27a80347 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java @@ -299,9 +299,9 @@ public void testCapacitySchedulerAbsoluteConfWithCustomResourceType() // Define top-level queues newConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); - newConf.setMinimumResourceRequirement("", "root.a", + newConf.setMinimumResourceRequirement("", new QueuePath("root", "a"), aMINRES); - newConf.setMaximumResourceRequirement("", "root.a", + newConf.setMaximumResourceRequirement("", new QueuePath("root", "a"), aMAXRES); newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index efa736d53c3bec..c3548cc6f7ecbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -676,12 +676,15 @@ private void nodeUpdate(NodeManager nm) { @Test public void testMaximumCapacitySetup() { float delta = 0.0000001f; + QueuePath queuePathA = new QueuePath(A); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta); + assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE, + conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta); conf.setMaximumCapacity(A, 50.0f); - assertEquals(50.0f, conf.getNonLabeledQueueMaximumCapacity(A),delta); + assertEquals(50.0f, conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta); conf.setMaximumCapacity(A, -1); - assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta); + assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE, + conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java index ad114d901cf9b3..1bee66eb9db9ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java @@ -60,14 +60,14 @@ public class TestCapacitySchedulerConfigValidator { private static final String LEAF_A = "leafA"; private static final String LEAF_B = "leafB"; - private static final String PARENT_A_FULL_PATH = CapacitySchedulerConfiguration.ROOT - + "." + PARENT_A; - private static final String LEAF_A_FULL_PATH = PARENT_A_FULL_PATH - + "." + LEAF_A; - private static final String PARENT_B_FULL_PATH = CapacitySchedulerConfiguration.ROOT - + "." + PARENT_B; - private static final String LEAF_B_FULL_PATH = PARENT_B_FULL_PATH - + "." + LEAF_B; + private static final QueuePath PARENT_A_FULL_PATH = + new QueuePath(CapacitySchedulerConfiguration.ROOT + "." + PARENT_A); + private static final QueuePath LEAF_A_FULL_PATH = + new QueuePath(PARENT_A_FULL_PATH + "." + LEAF_A); + private static final QueuePath PARENT_B_FULL_PATH = + new QueuePath(CapacitySchedulerConfiguration.ROOT + "." + PARENT_B); + private static final QueuePath LEAF_B_FULL_PATH = + new QueuePath(PARENT_B_FULL_PATH + "." + LEAF_B); private final Resource A_MINRES = Resource.newInstance(16 * GB, 10); private final Resource B_MINRES = Resource.newInstance(32 * GB, 5); @@ -225,7 +225,8 @@ public void testValidateCSConfigDefaultRCAbsoluteModeParentMaxMemoryExceeded() CapacitySchedulerConfiguration oldConfiguration = cs.getConfiguration(); CapacitySchedulerConfiguration newConfiguration = new CapacitySchedulerConfiguration(cs.getConfiguration()); - newConfiguration.setMaximumResourceRequirement("", LEAF_A_FULL_PATH, FULL_MAXRES); + newConfiguration.setMaximumResourceRequirement("", + LEAF_A_FULL_PATH, FULL_MAXRES); try { CapacitySchedulerConfigValidator .validateCSConfiguration(oldConfiguration, newConfiguration, rmContext); @@ -245,7 +246,8 @@ public void testValidateCSConfigDefaultRCAbsoluteModeParentMaxVcoreExceeded() th CapacitySchedulerConfiguration oldConfiguration = cs.getConfiguration(); CapacitySchedulerConfiguration newConfiguration = new CapacitySchedulerConfiguration(cs.getConfiguration()); - newConfiguration.setMaximumResourceRequirement("", LEAF_A_FULL_PATH, VCORE_EXCEEDED_MAXRES); + newConfiguration.setMaximumResourceRequirement("", + LEAF_A_FULL_PATH, VCORE_EXCEEDED_MAXRES); try { CapacitySchedulerConfigValidator .validateCSConfiguration(oldConfiguration, newConfiguration, rmContext); @@ -264,7 +266,8 @@ public void testValidateCSConfigDominantRCAbsoluteModeParentMaxMemoryExceeded() CapacitySchedulerConfiguration oldConfiguration = cs.getConfiguration(); CapacitySchedulerConfiguration newConfiguration = new CapacitySchedulerConfiguration(cs.getConfiguration()); - newConfiguration.setMaximumResourceRequirement("", LEAF_A_FULL_PATH, FULL_MAXRES); + newConfiguration.setMaximumResourceRequirement("", + LEAF_A_FULL_PATH, FULL_MAXRES); try { CapacitySchedulerConfigValidator .validateCSConfiguration(oldConfiguration, newConfiguration, rmContext); @@ -284,7 +287,8 @@ public void testValidateCSConfigDominantRCAbsoluteModeParentMaxVcoreExceeded() t CapacitySchedulerConfiguration oldConfiguration = cs.getConfiguration(); CapacitySchedulerConfiguration newConfiguration = new CapacitySchedulerConfiguration(cs.getConfiguration()); - newConfiguration.setMaximumResourceRequirement("", LEAF_A_FULL_PATH, VCORE_EXCEEDED_MAXRES); + newConfiguration.setMaximumResourceRequirement("", + LEAF_A_FULL_PATH, VCORE_EXCEEDED_MAXRES); try { CapacitySchedulerConfigValidator .validateCSConfiguration(oldConfiguration, newConfiguration, rmContext); @@ -304,7 +308,8 @@ public void testValidateCSConfigDominantRCAbsoluteModeParentMaxGPUExceeded() thr CapacitySchedulerConfiguration oldConfiguration = cs.getConfiguration(); CapacitySchedulerConfiguration newConfiguration = new CapacitySchedulerConfiguration(cs.getConfiguration()); - newConfiguration.setMaximumResourceRequirement("", LEAF_A_FULL_PATH, GPU_EXCEEDED_MAXRES_GPU); + newConfiguration.setMaximumResourceRequirement("", + LEAF_A_FULL_PATH, GPU_EXCEEDED_MAXRES_GPU); try { CapacitySchedulerConfigValidator .validateCSConfiguration(oldConfiguration, newConfiguration, rmContext); @@ -595,8 +600,8 @@ private CapacitySchedulerConfiguration setupCSConfiguration(YarnConfiguration co csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{PARENT_A, PARENT_B}); - csConf.setQueues(PARENT_A_FULL_PATH, new String[]{LEAF_A}); - csConf.setQueues(PARENT_B_FULL_PATH, new String[]{LEAF_B}); + csConf.setQueues(PARENT_A_FULL_PATH.getFullPath(), new String[]{LEAF_A}); + csConf.setQueues(PARENT_B_FULL_PATH.getFullPath(), new String[]{LEAF_B}); if (useDominantRC) { setupGpuResourceValues(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index fe90ca844359f5..31ece4f5f0f393 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -131,10 +131,10 @@ private void setupSingleLevelQueuesWithAbsoluteResource( // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B}); - conf.setMinimumResourceRequirement("", Q_A, + conf.setMinimumResourceRequirement("", new QueuePath(Q_A), QUEUE_A_RESOURCE); - conf.setMinimumResourceRequirement("", Q_B, + conf.setMinimumResourceRequirement("", new QueuePath(Q_B), QUEUE_B_RESOURCE); LOG.info("Setup top-level queues a and b with absolute resource"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java index bfbc0de31d335d..7eb577d9c12714 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java @@ -54,6 +54,16 @@ public void testEmptyPart() { Assert.assertFalse(queuePathWithoutEmptyPart.hasEmptyPart()); } + @Test + public void testNullPath() { + QueuePath queuePathWithNullPath = new QueuePath(null); + + Assert.assertNull(queuePathWithNullPath.getParent()); + Assert.assertEquals("", queuePathWithNullPath.getLeafName()); + Assert.assertEquals("", queuePathWithNullPath.getFullPath()); + Assert.assertFalse(queuePathWithNullPath.isRoot()); + } + @Test public void testIterator() { QueuePath queuePath = new QueuePath(TEST_QUEUE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 34b7c1225c2454..15599863d064a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; @@ -292,9 +293,9 @@ public void testAddNestedQueue() throws Exception { ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); assertEquals(4, newCSConf.getQueues("root").length); assertEquals(2, newCSConf.getQueues("root.d").length); - assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d1"), + assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.d.d1")), 0.01f); - assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d2"), + assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.d.d2")), 0.01f); CapacitySchedulerConfiguration newConf = getSchedulerConf(); @@ -330,8 +331,8 @@ public void testAddWithUpdate() throws Exception { CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); assertEquals(4, newCSConf.getQueues("root").length); - assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d"), 0.01f); - assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f); + assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.d")), 0.01f); + assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.01f); } @Test @@ -576,7 +577,7 @@ public void testRemoveParentQueueWithCapacity() throws Exception { CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); assertEquals(2, newCSConf.getQueues("root").length); - assertEquals(100.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), + assertEquals(100.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.01f); } @@ -718,8 +719,8 @@ public void testUpdateQueueCapacity() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.a"), 0.01f); - assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f); + assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.a")), 0.01f); + assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.01f); } @Test From 6d6da579b24ac4f4fb01481fded8fd7814cbe6b5 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 10 Dec 2021 17:14:04 +0800 Subject: [PATCH 04/33] HADOOP-17982. OpensslCipher initialization error should log a WARN message. (#3599) Change-Id: I070fc4784679b3be73aa3a11201bbae23c20ad4e --- .../src/main/java/org/apache/hadoop/crypto/OpensslCipher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java index d22e91442cca4e..0c65b74b2913bd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java @@ -84,14 +84,14 @@ static int get(String padding) throws NoSuchPaddingException { String loadingFailure = null; try { if (!NativeCodeLoader.buildSupportsOpenssl()) { - PerformanceAdvisory.LOG.debug("Build does not support openssl"); + PerformanceAdvisory.LOG.warn("Build does not support openssl"); loadingFailure = "build does not support openssl."; } else { initIDs(); } } catch (Throwable t) { loadingFailure = t.getMessage(); - LOG.debug("Failed to load OpenSSL Cipher.", t); + LOG.warn("Failed to load OpenSSL Cipher.", t); } finally { loadingFailureReason = loadingFailure; } From b60de1744b5cfec8eebd74aa5ed2722c5f552c50 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Fri, 10 Dec 2021 15:09:53 +0100 Subject: [PATCH 05/33] YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok --- .../webapp/dao/CapacitySchedulerInfo.java | 5 + .../dao/CapacitySchedulerQueueInfo.java | 12 +- ...WebServicesCapacitySchedDynamicConfig.java | 35 +- .../TestRMWebServicesForCSWithPartitions.java | 2 +- .../scheduler-response-AbsoluteMode.json | 3 +- ...sponse-AbsoluteModeLegacyAutoCreation.json | 1582 +++++++++++++++++ ...scheduler-response-NodeLabelDefaultAPI.xml | 3 +- .../scheduler-response-PerUserResources.json | 3 +- .../scheduler-response-PerUserResources.xml | 3 +- .../scheduler-response-PercentageMode.json | 3 +- ...onse-PercentageModeLegacyAutoCreation.json | 3 +- .../webapp/scheduler-response-WeightMode.json | 3 +- ...WeightModeWithAutoCreatedQueues-After.json | 3 +- ...eightModeWithAutoCreatedQueues-Before.json | 3 +- .../resources/webapp/scheduler-response.json | 3 +- .../resources/webapp/scheduler-response.xml | 3 +- 16 files changed, 1645 insertions(+), 24 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index c09ab5404c515e..74c7c2073b0e47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -49,6 +50,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected String queueName; private String queuePath; protected int maxParallelApps; + private boolean isAbsoluteResource; protected CapacitySchedulerQueueInfoList queues; protected QueueCapacitiesInfo capacities; protected CapacitySchedulerHealthInfo health; @@ -90,6 +92,9 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { health = new CapacitySchedulerHealthInfo(cs); maximumAllocation = new ResourceInfo(parent.getMaximumAllocation()); + isAbsoluteResource = parent.getCapacityConfigType() == + AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE; + CapacitySchedulerConfiguration conf = cs.getConfiguration(); queueAcls = new QueueAclsInfo(); queueAcls.addAll(getSortedQueueAclInfoList(queueName, conf)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index e27054d8be01b2..78b53922aafe8d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; @@ -43,11 +44,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity. - CapacitySchedulerConfiguration.RESOURCE_PATTERN; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity. - CapacitySchedulerConfiguration.CAPACITY; - @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) @XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class}) @@ -179,10 +175,8 @@ public class CapacitySchedulerQueueInfo { .getLeafOnlyProperties()); } - String configuredCapacity = conf.get( - CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY); - isAbsoluteResource = (configuredCapacity != null) - && RESOURCE_PATTERN.matcher(configuredCapacity).find(); + isAbsoluteResource = q.getCapacityConfigType() == + AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE; autoCreateChildQueueEnabled = conf. isAutoCreateChildQueueEnabled(queuePath); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index 1a87dd0bbfebe0..df4f18e1069ea1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -136,6 +136,22 @@ public void testSchedulerResponsePercentageModeLegacyAutoCreation() "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json"); } + @Test + public void testSchedulerResponseAbsoluteModeLegacyAutoCreation() + throws Exception { + Configuration config = CSConfigGenerator + .createAbsoluteConfigLegacyAutoCreation(); + config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); + + initResourceManager(config); + initAutoQueueHandler(8192 * GB); + createQueue("root.managed.queue1"); + + assertJsonResponse(sendRequest(), + "webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json"); + } + @Test public void testSchedulerResponseAbsoluteMode() throws Exception { @@ -189,7 +205,7 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues() "maximum-applications", 300); initResourceManager(config); - initAutoQueueHandler(); + initAutoQueueHandler(1200 * GB); // same as webapp/scheduler-response-WeightMode.json, but with effective resources filled in assertJsonResponse(sendRequest(), @@ -212,10 +228,10 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues() "webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json"); } - private void initAutoQueueHandler() throws Exception { + private void initAutoQueueHandler(int nodeMemory) throws Exception { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); autoQueueHandler = cs.getCapacitySchedulerQueueManager(); - rm.registerNode("h1:1234", 1200 * GB); // label = x + rm.registerNode("h1:1234", nodeMemory); // label = x } private void createQueue(String queuePath) throws YarnException, @@ -255,6 +271,19 @@ public static Configuration createPercentageConfigLegacyAutoCreation() { return createConfiguration(conf); } + public static Configuration createAbsoluteConfigLegacyAutoCreation() { + Map conf = new HashMap<>(); + conf.put("yarn.scheduler.capacity.root.queues", "default, managed"); + conf.put("yarn.scheduler.capacity.root.default.state", "STOPPED"); + conf.put("yarn.scheduler.capacity.root.managed.capacity", "[memory=4096,vcores=4]"); + conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.capacity", + "[memory=2048,vcores=2]"); + conf.put("yarn.scheduler.capacity.root.managed.state", "RUNNING"); + conf.put("yarn.scheduler.capacity.root.managed." + + "auto-create-child-queue.enabled", "true"); + return createConfiguration(conf); + } + public static Configuration createAbsoluteConfig() { Map conf = new HashMap<>(); conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index db9cbe6b20de78..0697ad03219521 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -574,7 +574,7 @@ private void verifySchedulerInfoJson(JSONObject json) JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements", 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements", 23, info.length()); + assertEquals("incorrect number of elements", 24, info.length()); JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES); JSONArray partitionsCapsArray = capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteMode.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteMode.json index 4909727f07b0b1..fb515d53914be7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteMode.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteMode.json @@ -8,6 +8,7 @@ "queueName": "root", "queuePath": "root", "maxParallelApps": 2147483647, + "isAbsoluteResource": true, "queues": {"queue": [ { "type": "capacitySchedulerLeafQueueInfo", @@ -1735,4 +1736,4 @@ "autoQueueTemplateProperties": {}, "autoQueueParentTemplateProperties": {}, "autoQueueLeafTemplateProperties": {} -}}} \ No newline at end of file +}}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json new file mode 100644 index 00000000000000..75800053980519 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json @@ -0,0 +1,1582 @@ +{"scheduler": {"schedulerInfo": { + "type": "capacityScheduler", + "capacity": 100, + "usedCapacity": 0, + "maxCapacity": 100, + "weight": -1, + "normalizedWeight": 0, + "queueName": "root", + "queuePath": "root", + "maxParallelApps": 2147483647, + "isAbsoluteResource": false, + "queues": {"queue": [ + { + "type": "capacitySchedulerLeafQueueInfo", + "queuePath": "root.default", + "capacity": 0, + "usedCapacity": 0, + "maxCapacity": 100, + "absoluteCapacity": 0, + "absoluteMaxCapacity": 100, + "absoluteUsedCapacity": 0, + "weight": -1, + "normalizedWeight": 0, + "numApplications": 0, + "maxParallelApps": 2147483647, + "queueName": "default", + "isAbsoluteResource": false, + "state": "STOPPED", + "resourcesUsed": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "hideReservationQueues": false, + "nodeLabels": ["*"], + "allocatedContainers": 0, + "reservedContainers": 0, + "pendingContainers": 0, + "capacities": {"queueCapacitiesByPartition": [{ + "partitionName": "", + "capacity": 0, + "usedCapacity": 0, + "maxCapacity": 100, + "absoluteCapacity": 0, + "absoluteUsedCapacity": 0, + "absoluteMaxCapacity": 100, + "maxAMLimitPercentage": 10, + "weight": -1, + "normalizedWeight": 0, + "configuredMinResource": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 8192, + "minimumAllocation": 1024, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 4, + "minimumAllocation": 1, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "configuredMaxResource": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 8192, + "minimumAllocation": 1024, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 4, + "minimumAllocation": 1, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "effectiveMinResource": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "effectiveMaxResource": { + "memory": 8388608, + "vCores": 8192, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 8388608 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 8192 + } + ]} + } + }]}, + "resources": {"resourceUsagesByPartition": [{ + "partitionName": "", + "used": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "reserved": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "pending": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "amUsed": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "amLimit": { + "memory": 839680, + "vCores": 1, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 839680 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 1 + } + ]} + }, + "userAmLimit": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + } + }]}, + "minEffectiveCapacity": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "maxEffectiveCapacity": { + "memory": 8388608, + "vCores": 8192, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 8388608 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 8192 + } + ]} + }, + "maximumAllocation": { + "memory": 8192, + "vCores": 4, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 8192 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 4 + } + ]} + }, + "queueAcls": {"queueAcl": [ + { + "accessType": "ADMINISTER_QUEUE", + "accessControlList": " " + }, + { + "accessType": "APPLICATION_MAX_PRIORITY", + "accessControlList": "*" + }, + { + "accessType": "SUBMIT_APP", + "accessControlList": " " + } + ]}, + "queuePriority": 0, + "orderingPolicyInfo": "fifo", + "autoCreateChildQueueEnabled": false, + "leafQueueTemplate": {}, + "mode": "percentage", + "queueType": "leaf", + "creationMethod": "static", + "autoCreationEligibility": "off", + "autoQueueTemplateProperties": {}, + "autoQueueParentTemplateProperties": {}, + "autoQueueLeafTemplateProperties": {}, + "numActiveApplications": 0, + "numPendingApplications": 0, + "numContainers": 0, + "maxApplications": 0, + "maxApplicationsPerUser": 0, + "userLimit": 100, + "users": {}, + "userLimitFactor": 1, + "configuredMaxAMResourceLimit": 0.1, + "AMResourceLimit": { + "memory": 839680, + "vCores": 1, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 839680 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 1 + } + ]} + }, + "usedAMResource": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "userAMResourceLimit": { + "memory": 839680, + "vCores": 1, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 839680 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 1 + } + ]} + }, + "preemptionDisabled": true, + "intraQueuePreemptionDisabled": true, + "defaultPriority": 0, + "isAutoCreatedLeafQueue": false, + "maxApplicationLifetime": -1, + "defaultApplicationLifetime": -1 + }, + { + "queuePath": "root.managed", + "capacity": 0.048828125, + "usedCapacity": 0, + "maxCapacity": 100, + "absoluteCapacity": 0.048828125, + "absoluteMaxCapacity": 100, + "absoluteUsedCapacity": 0, + "weight": -1, + "normalizedWeight": 0, + "numApplications": 0, + "maxParallelApps": 2147483647, + "queueName": "managed", + "isAbsoluteResource": true, + "state": "RUNNING", + "queues": {"queue": [{ + "type": "capacitySchedulerLeafQueueInfo", + "queuePath": "root.managed.queue1", + "capacity": 50, + "usedCapacity": 0, + "maxCapacity": 100, + "absoluteCapacity": 0.024414062, + "absoluteMaxCapacity": 100, + "absoluteUsedCapacity": 0, + "weight": -1, + "normalizedWeight": 0, + "numApplications": 0, + "maxParallelApps": 2147483647, + "queueName": "queue1", + "isAbsoluteResource": true, + "state": "RUNNING", + "resourcesUsed": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "hideReservationQueues": false, + "nodeLabels": ["*"], + "allocatedContainers": 0, + "reservedContainers": 0, + "pendingContainers": 0, + "capacities": {"queueCapacitiesByPartition": [{ + "partitionName": "", + "capacity": 50, + "usedCapacity": 0, + "maxCapacity": 100, + "absoluteCapacity": 0.024414062, + "absoluteUsedCapacity": 0, + "absoluteMaxCapacity": 100, + "maxAMLimitPercentage": 10, + "weight": -1, + "normalizedWeight": 0, + "configuredMinResource": { + "memory": 2048, + "vCores": 2, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 2048 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 2 + } + ]} + }, + "configuredMaxResource": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 8192, + "minimumAllocation": 1024, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 4, + "minimumAllocation": 1, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "effectiveMinResource": { + "memory": 2048, + "vCores": 2, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 2048 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 2 + } + ]} + }, + "effectiveMaxResource": { + "memory": 8388608, + "vCores": 8192, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 8388608 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 8192 + } + ]} + } + }]}, + "resources": {"resourceUsagesByPartition": [{ + "partitionName": "", + "used": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "reserved": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "pending": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "amUsed": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "amLimit": { + "memory": 839680, + "vCores": 1, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 839680 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 1 + } + ]} + }, + "userAmLimit": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + } + }]}, + "minEffectiveCapacity": { + "memory": 2048, + "vCores": 2, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 2048 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 2 + } + ]} + }, + "maxEffectiveCapacity": { + "memory": 8388608, + "vCores": 8192, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 8388608 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 8192 + } + ]} + }, + "maximumAllocation": { + "memory": 8192, + "vCores": 4, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 8192 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 4 + } + ]} + }, + "queueAcls": {"queueAcl": [ + { + "accessType": "ADMINISTER_QUEUE", + "accessControlList": " " + }, + { + "accessType": "APPLICATION_MAX_PRIORITY", + "accessControlList": "*" + }, + { + "accessType": "SUBMIT_APP", + "accessControlList": " " + } + ]}, + "queuePriority": 0, + "orderingPolicyInfo": "fifo", + "autoCreateChildQueueEnabled": false, + "leafQueueTemplate": {}, + "mode": "absolute", + "queueType": "leaf", + "creationMethod": "dynamicLegacy", + "autoCreationEligibility": "off", + "autoQueueTemplateProperties": {}, + "autoQueueParentTemplateProperties": {}, + "autoQueueLeafTemplateProperties": {}, + "numActiveApplications": 0, + "numPendingApplications": 0, + "numContainers": 0, + "maxApplications": 2, + "maxApplicationsPerUser": 2, + "userLimit": 100, + "users": {}, + "userLimitFactor": 1, + "configuredMaxAMResourceLimit": 0.1, + "AMResourceLimit": { + "memory": 839680, + "vCores": 1, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 839680 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 1 + } + ]} + }, + "usedAMResource": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "userAMResourceLimit": { + "memory": 839680, + "vCores": 1, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 839680 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 1 + } + ]} + }, + "preemptionDisabled": true, + "intraQueuePreemptionDisabled": true, + "defaultPriority": 0, + "isAutoCreatedLeafQueue": true, + "maxApplicationLifetime": -1, + "defaultApplicationLifetime": -1 + }]}, + "resourcesUsed": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "hideReservationQueues": false, + "nodeLabels": ["*"], + "allocatedContainers": 0, + "reservedContainers": 0, + "pendingContainers": 0, + "capacities": {"queueCapacitiesByPartition": [{ + "partitionName": "", + "capacity": 0.048828125, + "usedCapacity": 0, + "maxCapacity": 100, + "absoluteCapacity": 0.048828125, + "absoluteUsedCapacity": 0, + "absoluteMaxCapacity": 100, + "maxAMLimitPercentage": 0, + "weight": -1, + "normalizedWeight": 0, + "configuredMinResource": { + "memory": 4096, + "vCores": 4, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 4096 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 4 + } + ]} + }, + "configuredMaxResource": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 8192, + "minimumAllocation": 1024, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 4, + "minimumAllocation": 1, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "effectiveMinResource": { + "memory": 4096, + "vCores": 4, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 4096 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 4 + } + ]} + }, + "effectiveMaxResource": { + "memory": 8388608, + "vCores": 8192, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 8388608 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 8192 + } + ]} + } + }]}, + "resources": {"resourceUsagesByPartition": [{ + "partitionName": "", + "used": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "reserved": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "pending": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + } + }]}, + "minEffectiveCapacity": { + "memory": 4096, + "vCores": 4, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 4096 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 4 + } + ]} + }, + "maxEffectiveCapacity": { + "memory": 8388608, + "vCores": 8192, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 8388608 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 8192 + } + ]} + }, + "maximumAllocation": { + "memory": 8192, + "vCores": 4, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 8192 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 4 + } + ]} + }, + "queueAcls": {"queueAcl": [ + { + "accessType": "ADMINISTER_QUEUE", + "accessControlList": " " + }, + { + "accessType": "APPLICATION_MAX_PRIORITY", + "accessControlList": "*" + }, + { + "accessType": "SUBMIT_APP", + "accessControlList": " " + } + ]}, + "queuePriority": 0, + "orderingPolicyInfo": "utilization", + "autoCreateChildQueueEnabled": true, + "leafQueueTemplate": {"property": [{ + "name": "leaf-queue-template.capacity", + "value": "[memory=2048,vcores=2]" + }]}, + "mode": "absolute", + "queueType": "parent", + "creationMethod": "static", + "autoCreationEligibility": "legacy", + "autoQueueTemplateProperties": {}, + "autoQueueParentTemplateProperties": {}, + "autoQueueLeafTemplateProperties": {} + } + ]}, + "capacities": {"queueCapacitiesByPartition": [{ + "partitionName": "", + "capacity": 100, + "usedCapacity": 0, + "maxCapacity": 100, + "absoluteCapacity": 100, + "absoluteUsedCapacity": 0, + "absoluteMaxCapacity": 100, + "maxAMLimitPercentage": 0, + "weight": -1, + "normalizedWeight": 0, + "configuredMinResource": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 8192, + "minimumAllocation": 1024, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 4, + "minimumAllocation": 1, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "configuredMaxResource": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 8192, + "minimumAllocation": 1024, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 4, + "minimumAllocation": 1, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + }, + "effectiveMinResource": { + "memory": 8388608, + "vCores": 8192, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 8388608 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 8192 + } + ]} + }, + "effectiveMaxResource": { + "memory": 8388608, + "vCores": 8192, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 8388608 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 8192 + } + ]} + } + }]}, + "health": { + "lastrun": 0, + "operationsInfo": [ + { + "operation": "last-allocation", + "nodeId": "N\/A", + "containerId": "N\/A", + "queue": "N\/A" + }, + { + "operation": "last-release", + "nodeId": "N\/A", + "containerId": "N\/A", + "queue": "N\/A" + }, + { + "operation": "last-preemption", + "nodeId": "N\/A", + "containerId": "N\/A", + "queue": "N\/A" + }, + { + "operation": "last-reservation", + "nodeId": "N\/A", + "containerId": "N\/A", + "queue": "N\/A" + } + ], + "lastRunDetails": [ + { + "operation": "releases", + "count": 0, + "resources": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + } + }, + { + "operation": "allocations", + "count": 0, + "resources": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + } + }, + { + "operation": "reservations", + "count": 0, + "resources": { + "memory": 0, + "vCores": 0, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 0 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 0 + } + ]} + } + } + ] + }, + "maximumAllocation": { + "memory": 8192, + "vCores": 4, + "resourceInformations": {"resourceInformation": [ + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "memory-mb", + "resourceType": "COUNTABLE", + "units": "Mi", + "value": 8192 + }, + { + "attributes": {}, + "maximumAllocation": 9223372036854775807, + "minimumAllocation": 0, + "name": "vcores", + "resourceType": "COUNTABLE", + "units": "", + "value": 4 + } + ]} + }, + "queueAcls": {"queueAcl": [ + { + "accessType": "ADMINISTER_QUEUE", + "accessControlList": "*" + }, + { + "accessType": "APPLICATION_MAX_PRIORITY", + "accessControlList": "*" + }, + { + "accessType": "SUBMIT_APP", + "accessControlList": "*" + } + ]}, + "queuePriority": 0, + "orderingPolicyInfo": "utilization", + "mode": "percentage", + "queueType": "parent", + "creationMethod": "static", + "autoCreationEligibility": "off", + "autoQueueTemplateProperties": {}, + "autoQueueParentTemplateProperties": {}, + "autoQueueLeafTemplateProperties": {} +}}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml index ac51fcf3106047..48b6893ac96029 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml @@ -8,6 +8,7 @@ root root 2147483647 + false root.a @@ -4546,4 +4547,4 @@ - \ No newline at end of file + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json index bbf127f146ef21..7960bf73eb0acb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json @@ -8,6 +8,7 @@ "queueName": "root", "queuePath": "root", "maxParallelApps": 2147483647, + "isAbsoluteResource": false, "queues": {"queue": [ { "queuePath": "root.a", @@ -4953,4 +4954,4 @@ "autoQueueTemplateProperties": {}, "autoQueueParentTemplateProperties": {}, "autoQueueLeafTemplateProperties": {} -}}} \ No newline at end of file +}}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml index 0e4d152429c252..97d937b7f4c54f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml @@ -8,6 +8,7 @@ root root 2147483647 + false root.a @@ -4992,4 +4993,4 @@ - \ No newline at end of file + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageMode.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageMode.json index 71fe8e96266285..cb3441a7aae0fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageMode.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageMode.json @@ -8,6 +8,7 @@ "queueName": "root", "queuePath": "root", "maxParallelApps": 2147483647, + "isAbsoluteResource": false, "queues": {"queue": [ { "type": "capacitySchedulerLeafQueueInfo", @@ -1735,4 +1736,4 @@ "autoQueueTemplateProperties": {}, "autoQueueParentTemplateProperties": {}, "autoQueueLeafTemplateProperties": {} -}}} \ No newline at end of file +}}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageModeLegacyAutoCreation.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageModeLegacyAutoCreation.json index 3abe605b7e50f8..8fed362b0291a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageModeLegacyAutoCreation.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageModeLegacyAutoCreation.json @@ -8,6 +8,7 @@ "queueName": "root", "queuePath": "root", "maxParallelApps": 2147483647, + "isAbsoluteResource": false, "queues": {"queue": [ { "type": "capacitySchedulerLeafQueueInfo", @@ -1576,4 +1577,4 @@ "autoQueueTemplateProperties": {}, "autoQueueParentTemplateProperties": {}, "autoQueueLeafTemplateProperties": {} -}}} \ No newline at end of file +}}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightMode.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightMode.json index bb230a7ef5134b..b1894c58ad7f01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightMode.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightMode.json @@ -8,6 +8,7 @@ "queueName": "root", "queuePath": "root", "maxParallelApps": 2147483647, + "isAbsoluteResource": false, "queues": {"queue": [ { "type": "capacitySchedulerLeafQueueInfo", @@ -1735,4 +1736,4 @@ "autoQueueTemplateProperties": {}, "autoQueueParentTemplateProperties": {}, "autoQueueLeafTemplateProperties": {} -}}} \ No newline at end of file +}}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json index 9ab65fb6abe942..188b72a7a405db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json @@ -8,6 +8,7 @@ "queueName": "root", "queuePath": "root", "maxParallelApps": 2147483647, + "isAbsoluteResource": false, "queues": {"queue": [ { "type": "capacitySchedulerLeafQueueInfo", @@ -4003,4 +4004,4 @@ "autoQueueTemplateProperties": {}, "autoQueueParentTemplateProperties": {}, "autoQueueLeafTemplateProperties": {} -}}} \ No newline at end of file +}}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-Before.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-Before.json index a2b6acfe3ac155..ee4cd14a1b3fab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-Before.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-Before.json @@ -8,6 +8,7 @@ "queueName": "root", "queuePath": "root", "maxParallelApps": 2147483647, + "isAbsoluteResource": false, "queues": {"queue": [ { "type": "capacitySchedulerLeafQueueInfo", @@ -1735,4 +1736,4 @@ "autoQueueTemplateProperties": {}, "autoQueueParentTemplateProperties": {}, "autoQueueLeafTemplateProperties": {} -}}} \ No newline at end of file +}}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json index eed784acbe9d39..26289cceda7bb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json @@ -8,6 +8,7 @@ "queueName": "root", "queuePath": "root", "maxParallelApps": 2147483647, + "isAbsoluteResource": false, "queues": {"queue": [ { "queuePath": "root.a", @@ -4500,4 +4501,4 @@ "autoQueueTemplateProperties": {}, "autoQueueParentTemplateProperties": {}, "autoQueueLeafTemplateProperties": {} -}}} \ No newline at end of file +}}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml index 5c0a3de5354cc9..d196ec8be61670 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml @@ -8,6 +8,7 @@ root root 2147483647 + false root.a @@ -4535,4 +4536,4 @@ - \ No newline at end of file + From f5d7192528906695a3a1bdbe0e2328da194a682f Mon Sep 17 00:00:00 2001 From: better3471 <46600375+better3471@users.noreply.github.com> Date: Mon, 13 Dec 2021 09:45:47 +0800 Subject: [PATCH 06/33] HADOOP-18042. Fix jetty version in LICENSE-binary (#3783) Signed-off-by: Akira Ajisaka --- LICENSE-binary | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 07dd67a22abc46..6dabe3d71de040 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -328,20 +328,20 @@ org.codehaus.jackson:jackson-jaxrs:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.jackson:jackson-xc:1.9.13 org.codehaus.jettison:jettison:1.1 -org.eclipse.jetty:jetty-annotations:9.3.27.v20190418 -org.eclipse.jetty:jetty-http:9.3.27.v20190418 -org.eclipse.jetty:jetty-io:9.3.27.v20190418 -org.eclipse.jetty:jetty-jndi:9.3.27.v20190418 -org.eclipse.jetty:jetty-plus:9.3.27.v20190418 -org.eclipse.jetty:jetty-security:9.3.27.v20190418 -org.eclipse.jetty:jetty-server:9.3.27.v20190418 -org.eclipse.jetty:jetty-servlet:9.3.27.v20190418 -org.eclipse.jetty:jetty-util:9.3.27.v20190418 -org.eclipse.jetty:jetty-util-ajax:9.3.27.v20190418 -org.eclipse.jetty:jetty-webapp:9.3.27.v20190418 -org.eclipse.jetty:jetty-xml:9.3.27.v20190418 -org.eclipse.jetty.websocket:javax-websocket-client-impl:9.3.27.v20190418 -org.eclipse.jetty.websocket:javax-websocket-server-impl:9.3.27.v20190418 +org.eclipse.jetty:jetty-annotations:9.4.44.v20210927 +org.eclipse.jetty:jetty-http:9.4.44.v20210927 +org.eclipse.jetty:jetty-io:9.4.44.v20210927 +org.eclipse.jetty:jetty-jndi:9.4.44.v20210927 +org.eclipse.jetty:jetty-plus:9.4.44.v20210927 +org.eclipse.jetty:jetty-security:9.4.44.v20210927 +org.eclipse.jetty:jetty-server:9.4.44.v20210927 +org.eclipse.jetty:jetty-servlet:9.4.44.v20210927 +org.eclipse.jetty:jetty-util:9.4.44.v20210927 +org.eclipse.jetty:jetty-util-ajax:9.4.44.v20210927 +org.eclipse.jetty:jetty-webapp:9.4.44.v20210927 +org.eclipse.jetty:jetty-xml:9.4.44.v20210927 +org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.44.v20210927 +org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.44.v20210927 org.ehcache:ehcache:3.3.1 org.lz4:lz4-java:1.7.1 org.objenesis:objenesis:2.6 From 08041e4d400bfd44807c9885cb8f911c67983e89 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 13 Dec 2021 10:33:32 +0530 Subject: [PATCH 07/33] HADOOP-18039. Upgrade hbase2 version and fix TestTimelineWriterHBaseDown (#3768) Signed-off-by: Akira Ajisaka --- hadoop-project/pom.xml | 5 ++-- .../pom.xml | 24 +++++++++++++++++++ .../storage/TestTimelineWriterHBaseDown.java | 2 +- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 62e047254b7077..ec64857ed231b9 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -200,7 +200,7 @@ 1.5.4 1.26 1.7.1 - 2.0.2 + 2.2.4 4.13.2 5.5.1 5.5.1 @@ -2427,9 +2427,10 @@ ${hbase.two.version} - 3.0.0 + 2.8.5 11.0.2 hadoop-yarn-server-timelineservice-hbase-server-2 + 9.3.27.v20190418 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml index 8eebb782da8f7e..fee962575e80c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml @@ -515,6 +515,30 @@ mockito-core test + + org.eclipse.jetty + jetty-server + test + ${hbase-compatible-jetty.version} + + + org.eclipse.jetty + jetty-servlet + test + ${hbase-compatible-jetty.version} + + + org.eclipse.jetty + jetty-webapp + test + ${hbase-compatible-jetty.version} + + + org.eclipse.jetty + jetty-util + test + ${hbase-compatible-jetty.version} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterHBaseDown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterHBaseDown.java index cb89ba4223e6c3..5d658b96c5b1c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterHBaseDown.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterHBaseDown.java @@ -43,12 +43,12 @@ public void testTimelineWriterHBaseDown() throws Exception { HBaseTestingUtility util = new HBaseTestingUtility(); HBaseTimelineWriterImpl writer = new HBaseTimelineWriterImpl(); try { + util.startMiniCluster(); Configuration c1 = util.getConfiguration(); c1.setLong(TIMELINE_SERVICE_READER_STORAGE_MONITOR_INTERVAL_MS, 5000); writer.init(c1); writer.start(); - util.startMiniCluster(); DataGeneratorForTest.createSchema(util.getConfiguration()); TimelineStorageMonitor storageMonitor = writer. From b86083b935d594013ae933643812e5370d9f1e94 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Mon, 13 Dec 2021 17:49:22 +0900 Subject: [PATCH 08/33] HADOOP-18043. Use mina-core 2.0.22 to fix LDAP unit test failures (#3792) Reviewed-by: Ayush Saxena --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index ec64857ed231b9..cc45975a3f0771 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1015,7 +1015,7 @@ org.apache.mina mina-core - 2.1.5 + 2.0.22 org.apache.sshd From 5084d03aa06fe387adc5844e9b6faca69dc2b84e Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Mon, 13 Dec 2021 16:12:55 +0100 Subject: [PATCH 09/33] YARN-11024. Create an AbstractLeafQueue to store the common LeafQueue + AutoCreatedLeafQueue functionality. Contributed by Benjamin Teke --- .../metrics/TimelineServiceV1Publisher.java | 30 +- .../capacity/FifoCandidatesSelector.java | 4 +- .../FifoIntraQueuePreemptionPlugin.java | 4 +- .../IntraQueueCandidatesSelector.java | 13 +- .../capacity/TempQueuePerPartition.java | 10 +- .../placement/CSMappingPlacementRule.java | 4 +- .../placement/QueuePlacementRuleUtils.java | 6 +- .../MappingRuleValidationContextImpl.java | 6 +- .../AbstractAutoCreatedLeafQueue.java | 2 +- .../scheduler/capacity/AbstractLeafQueue.java | 2382 ++++++++++++++++ .../capacity/AutoCreatedLeafQueue.java | 6 +- .../capacity/CSMaxRunningAppsEnforcer.java | 8 +- .../capacity/CapacityHeadroomProvider.java | 8 +- .../scheduler/capacity/CapacityScheduler.java | 44 +- .../CapacitySchedulerConfigValidator.java | 4 +- .../CapacitySchedulerQueueManager.java | 14 +- .../scheduler/capacity/LeafQueue.java | 2436 +---------------- .../capacity/ManagedParentQueue.java | 6 +- .../scheduler/capacity/ParentQueue.java | 8 +- .../scheduler/capacity/ReservationQueue.java | 5 + .../scheduler/capacity/UsersManager.java | 4 +- ...uaranteedOrZeroCapacityOverTimePolicy.java | 50 +- .../common/fica/FiCaSchedulerApp.java | 10 +- .../webapp/dao/CapacitySchedulerInfo.java | 8 +- .../dao/CapacitySchedulerLeafQueueInfo.java | 4 +- .../helper/CapacitySchedulerInfoHelper.java | 4 +- .../TestWorkPreservingRMRestart.java | 5 +- .../applicationsmanager/TestAMRestart.java | 1 - .../TestCombinedSystemMetricsPublisher.java | 2 - .../metrics/TestSystemMetricsPublisher.java | 45 +- .../TestCSMaxRunningAppsEnforcer.java | 2 +- ...CapacitySchedulerNewQueueAutoCreation.java | 15 +- 32 files changed, 2561 insertions(+), 2589 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java index 86576f7b1d5432..f1b80a946a7d0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java @@ -86,14 +86,26 @@ protected void serviceInit(Configuration conf) throws Exception { conf.getInt(YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL, YarnConfiguration.DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL) * 1000; + if (putEventInterval <= 0) { + throw new IllegalArgumentException( + "RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL should be greater than 0"); + } dispatcherPoolSize = conf.getInt( YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE, YarnConfiguration. DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE); + if (dispatcherPoolSize <= 0) { + throw new IllegalArgumentException( + "RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE should be greater than 0"); + } dispatcherBatchSize = conf.getInt( YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE, YarnConfiguration. DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE); + if (dispatcherBatchSize <= 1) { + throw new IllegalArgumentException( + "RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE should be greater than 1"); + } putEventThread = new PutEventThread(); sendEventThreadPool = Executors.newFixedThreadPool(dispatcherPoolSize); entityQueue = new LinkedBlockingQueue<>(dispatcherBatchSize + 1); @@ -126,7 +138,8 @@ protected void serviceStop() throws Exception { putEventThread.join(); SendEntity task = new SendEntity(); if (!task.buffer.isEmpty()) { - LOG.info(String.format("Initiating final putEntities, remaining entities left in entityQueue: %d", task.buffer.size())); + LOG.info("Initiating final putEntities, remaining entities left in entityQueue: {}", + task.buffer.size()); sendEventThreadPool.submit(task); } } finally { @@ -461,8 +474,7 @@ private void putEntity(TimelineEntity entity) { LOG.error("Error when publishing entity batch [ " + entity.getEntityType() + "," + entity.getEntityId() + " ] ", e); } - } - else { + } else { try { if (LOG.isDebugEnabled()) { LOG.debug("Publishing the entity " + entity.getEntityId() @@ -481,7 +493,7 @@ private class SendEntity implements Runnable { private ArrayList buffer; - public SendEntity(){ + SendEntity() { buffer = new ArrayList(); entityQueue.drainTo(buffer); } @@ -489,7 +501,7 @@ public SendEntity(){ @Override public void run() { if (LOG.isDebugEnabled()) { - LOG.debug(String.format("Number of timeline entities being sent in batch: %d", buffer.size())); + LOG.debug("Number of timeline entities being sent in batch: {}", buffer.size()); } if (buffer.isEmpty()) { return; @@ -505,7 +517,7 @@ public void run() { private class TimelineV1PublishEvent extends TimelinePublishEvent { private TimelineEntity entity; - public TimelineV1PublishEvent(SystemMetricsEventType type, + TimelineV1PublishEvent(SystemMetricsEventType type, TimelineEntity entity, ApplicationId appId) { super(type, appId); this.entity = entity; @@ -525,7 +537,7 @@ public void handle(TimelineV1PublishEvent event) { } private class PutEventThread extends Thread { - public PutEventThread() { + PutEventThread() { super("PutEventThread"); } @@ -565,6 +577,4 @@ public void run() { } } } -} - - +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java index d9e9091bc86f70..440066c5ca606d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java @@ -24,7 +24,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; @@ -86,7 +86,7 @@ public Map> selectCandidates( } // compute resToObtainByPartition considered inter-queue preemption - LeafQueue leafQueue = preemptionContext.getQueueByPartition(queueName, + AbstractLeafQueue leafQueue = preemptionContext.getQueueByPartition(queueName, RMNodeLabelsManager.NO_LABEL).leafQueue; Map resToObtainByPartition = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java index ea17feda8c3f7c..188e619891ee7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java @@ -40,7 +40,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.IntraQueuePreemptionOrderPolicy; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy; @@ -577,7 +577,7 @@ public void validateOutSameAppPriorityFromDemand(Resource cluster, } private Resource calculateUsedAMResourcesPerQueue(String partition, - LeafQueue leafQueue, Map perUserAMUsed) { + AbstractLeafQueue leafQueue, Map perUserAMUsed) { Collection runningApps = leafQueue.getApplications(); Resource amUsed = Resources.createResource(0, 0); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java index cea1bca7736900..f0bd03b7d13300 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java @@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.IntraQueuePreemptionOrderPolicy; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.AbstractComparatorOrderingPolicy; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -146,7 +146,7 @@ public Map> selectCandidates( // 4. Iterate from most under-served queue in order. for (String queueName : queueNames) { - LeafQueue leafQueue = preemptionContext.getQueueByPartition(queueName, + AbstractLeafQueue leafQueue = preemptionContext.getQueueByPartition(queueName, RMNodeLabelsManager.NO_LABEL).leafQueue; // skip if not a leafqueue @@ -181,7 +181,7 @@ public Map> selectCandidates( leafQueue.getReadLock().lock(); try { for (FiCaSchedulerApp app : apps) { - preemptFromLeastStarvedApp(leafQueue, app, selectedCandidates, + preemptFromLeastStarvedApp(app, selectedCandidates, curCandidates, clusterResource, totalPreemptedResourceAllowed, resToObtainByPartition, rollingResourceUsagePerUser); } @@ -195,7 +195,7 @@ public Map> selectCandidates( } private void initializeUsageAndUserLimitForCompute(Resource clusterResource, - String partition, LeafQueue leafQueue, + String partition, AbstractLeafQueue leafQueue, Map rollingResourceUsagePerUser) { for (String user : leafQueue.getAllUsers()) { // Initialize used resource of a given user for rolling computation. @@ -206,8 +206,7 @@ private void initializeUsageAndUserLimitForCompute(Resource clusterResource, } } - private void preemptFromLeastStarvedApp(LeafQueue leafQueue, - FiCaSchedulerApp app, + private void preemptFromLeastStarvedApp(FiCaSchedulerApp app, Map> selectedCandidates, Map> curCandidates, Resource clusterResource, Resource totalPreemptedResourceAllowed, @@ -293,7 +292,7 @@ private void computeIntraQueuePreemptionDemand(Resource clusterResource, for (String queueName : queueNames) { TempQueuePerPartition tq = context.getQueueByPartition(queueName, partition); - LeafQueue leafQueue = tq.leafQueue; + AbstractLeafQueue leafQueue = tq.leafQueue; // skip if its parent queue if (null == leafQueue) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java index 57dc6395702154..958c08e8038af6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java @@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceUtils; @@ -56,7 +56,7 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity { final ArrayList children; private Collection apps; - LeafQueue leafQueue; + AbstractLeafQueue leafQueue; ParentQueue parentQueue; boolean preemptionDisabled; @@ -81,8 +81,8 @@ public TempQueuePerPartition(String queueName, Resource current, super(queueName, current, Resource.newInstance(0, 0), reserved, Resource.newInstance(0, 0)); - if (queue instanceof LeafQueue) { - LeafQueue l = (LeafQueue) queue; + if (queue instanceof AbstractLeafQueue) { + AbstractLeafQueue l = (AbstractLeafQueue) queue; pending = l.getTotalPendingResourcesConsideringUserLimit( totalPartitionResource, partition, false); pendingDeductReserved = l.getTotalPendingResourcesConsideringUserLimit( @@ -113,7 +113,7 @@ public TempQueuePerPartition(String queueName, Resource current, this.effMaxRes = effMaxRes; } - public void setLeafQueue(LeafQueue l) { + public void setLeafQueue(AbstractLeafQueue l) { assert children.size() == 0; this.leafQueue = l; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java index d9c7e6f073b590..cefed1dd9fd85a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java @@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -277,7 +277,7 @@ private String validateAndNormalizeQueue( } CSQueue queue = queueManager.getQueueByFullName(normalizedName); - if (queue != null && !(queue instanceof LeafQueue)) { + if (queue != null && !(queue instanceof AbstractLeafQueue)) { throw new YarnException("Mapping rule returned a non-leaf queue '" + normalizedName + "', cannot place application in it."); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java index 76e3e275fc9f7a..f6381bdcc1f72b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java @@ -21,13 +21,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue; import java.io.IOException; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; - /** * Utility class for Capacity Scheduler queue PlacementRules. */ @@ -83,7 +81,7 @@ public static QueueMapping validateAndGetAutoCreatedQueueMapping( public static QueueMapping validateAndGetQueueMapping( CapacitySchedulerQueueManager queueManager, CSQueue queue, QueueMapping mapping) throws IOException { - if (!(queue instanceof LeafQueue)) { + if (!(queue instanceof AbstractLeafQueue)) { throw new IOException( "mapping contains invalid or non-leaf queue : " + mapping.getFullPath()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/MappingRuleValidationContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/MappingRuleValidationContextImpl.java index 4218b6faa24787..cceb7e6bb2315f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/MappingRuleValidationContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/MappingRuleValidationContextImpl.java @@ -22,7 +22,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; @@ -91,7 +91,7 @@ private boolean validateStaticQueuePath(QueuePath path) "' under it."); case QUEUE_EXISTS: CSQueue queue = queueManager.getQueue(normalizedPath); - if (!(queue instanceof LeafQueue)) { + if (!(queue instanceof AbstractLeafQueue)) { throw new YarnException("Target queue '" + path.getFullPath() + "' but it's not a leaf queue."); } @@ -157,7 +157,7 @@ private boolean validateDynamicQueuePath(QueuePath path) //if the static part of our queue exists, and it's not a leaf queue, //we cannot do any deeper validation if (queue != null) { - if (queue instanceof LeafQueue) { + if (queue instanceof AbstractLeafQueue) { throw new YarnException("Queue path '" + path +"' is invalid " + "because '" + normalizedStaticPart + "' is a leaf queue, " + "which can have no other queues under it."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java index b9c2ec62364903..36d2aef4806ed8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java @@ -34,7 +34,7 @@ * Abstract class for dynamic auto created queues managed by an implementation * of AbstractManagedParentQueue */ -public class AbstractAutoCreatedLeafQueue extends LeafQueue { +public class AbstractAutoCreatedLeafQueue extends AbstractLeafQueue { protected AbstractManagedParentQueue parent; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java new file mode 100644 index 00000000000000..99911400f26e44 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java @@ -0,0 +1,2382 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.util.Sets; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerExitStatus; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.ExecutionType; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesLogger; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerAllocationProposal; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyForPendingApps; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.IteratorSelector; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy; +import org.apache.hadoop.yarn.server.utils.Lock; +import org.apache.hadoop.yarn.server.utils.Lock.NoLock; +import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.hadoop.yarn.util.resource.Resources; + +import org.apache.hadoop.classification.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AbstractLeafQueue extends AbstractCSQueue { + private static final Logger LOG = + LoggerFactory.getLogger(AbstractLeafQueue.class); + + private float absoluteUsedCapacity = 0.0f; + + // TODO the max applications should consider label + protected int maxApplications; + protected volatile int maxApplicationsPerUser; + + private float maxAMResourcePerQueuePercent; + + private volatile int nodeLocalityDelay; + private volatile int rackLocalityAdditionalDelay; + private volatile boolean rackLocalityFullReset; + + Map applicationAttemptMap = + new ConcurrentHashMap<>(); + + private Priority defaultAppPriorityPerQueue; + + private final OrderingPolicy pendingOrderingPolicy; + + private volatile float minimumAllocationFactor; + + private final RecordFactory recordFactory = + RecordFactoryProvider.getRecordFactory(null); + + private final UsersManager usersManager; + + // cache last cluster resource to compute actual capacity + private Resource lastClusterResource = Resources.none(); + + private final QueueResourceLimitsInfo queueResourceLimitsInfo = + new QueueResourceLimitsInfo(); + + private volatile ResourceLimits cachedResourceLimitsForHeadroom = null; + + private volatile OrderingPolicy orderingPolicy = null; + + // Map>> + // Not thread safe: only the last level is a ConcurrentMap + @VisibleForTesting + Map>> + userLimitsCache = new HashMap<>(); + + // Not thread safe + @VisibleForTesting + long currentUserLimitCacheVersion = 0; + + // record all ignore partition exclusivityRMContainer, this will be used to do + // preemption, key is the partition of the RMContainer allocated on + private Map> ignorePartitionExclusivityRMContainers = + new ConcurrentHashMap<>(); + + List priorityAcls = + new ArrayList(); + + private final List runnableApps = new ArrayList<>(); + private final List nonRunnableApps = new ArrayList<>(); + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public AbstractLeafQueue(CapacitySchedulerContext cs, String queueName, + CSQueue parent, CSQueue old) { + this(cs, cs.getConfiguration(), queueName, parent, old, false); + } + + public AbstractLeafQueue(CapacitySchedulerContext cs, CapacitySchedulerConfiguration configuration, + String queueName, CSQueue parent, CSQueue old) { + this(cs, configuration, queueName, parent, old, false); + } + + public AbstractLeafQueue(CapacitySchedulerContext cs, CapacitySchedulerConfiguration configuration, + String queueName, CSQueue parent, CSQueue old, boolean isDynamic) { + super(cs, configuration, queueName, parent, old); + setDynamicQueue(isDynamic); + + this.usersManager = new UsersManager(usageTracker.getMetrics(), this, labelManager, csContext, + resourceCalculator); + + // One time initialization is enough since it is static ordering policy + this.pendingOrderingPolicy = new FifoOrderingPolicyForPendingApps(); + } + + @SuppressWarnings("checkstyle:nowhitespaceafter") + protected void setupQueueConfigs( + Resource clusterResource, CapacitySchedulerConfiguration conf) throws IOException { + writeLock.lock(); + try { + CapacitySchedulerConfiguration schedConf = csContext.getConfiguration(); + super.setupQueueConfigs(clusterResource, conf); + + this.lastClusterResource = clusterResource; + + this.cachedResourceLimitsForHeadroom = new ResourceLimits(clusterResource); + + // Initialize headroom info, also used for calculating application + // master resource limits. Since this happens during queue initialization + // and all queues may not be realized yet, we'll use (optimistic) + // absoluteMaxCapacity (it will be replaced with the more accurate + // absoluteMaxAvailCapacity during headroom/userlimit/allocation events) + setQueueResourceLimitsInfo(clusterResource); + + setOrderingPolicy(conf.getAppOrderingPolicy(getQueuePath())); + + usersManager.setUserLimit(conf.getUserLimit(getQueuePath())); + usersManager.setUserLimitFactor(conf.getUserLimitFactor(getQueuePath())); + + maxAMResourcePerQueuePercent = + conf.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath()); + + maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath()); + if (maxApplications < 0) { + int maxGlobalPerQueueApps = + csContext.getConfiguration().getGlobalMaximumApplicationsPerQueue(); + if (maxGlobalPerQueueApps > 0) { + maxApplications = maxGlobalPerQueueApps; + } + } + + priorityAcls = + conf.getPriorityAcls(getQueuePath(), csContext.getMaxClusterLevelAppPriority()); + + Set accessibleNodeLabels = this.queueNodeLabelsSettings.getAccessibleNodeLabels(); + if (!SchedulerUtils.checkQueueLabelExpression(accessibleNodeLabels, + this.queueNodeLabelsSettings.getDefaultLabelExpression(), null)) { + throw new IOException("Invalid default label expression of " + " queue=" + getQueuePath() + + " doesn't have permission to access all labels " + + "in default label expression. labelExpression of resource request=" + + getDefaultNodeLabelExpressionStr() + ". Queue labels=" + ( + getAccessibleNodeLabels() == null ? "" : + StringUtils.join(getAccessibleNodeLabels().iterator(), ','))); + } + + nodeLocalityDelay = schedConf.getNodeLocalityDelay(); + rackLocalityAdditionalDelay = schedConf.getRackLocalityAdditionalDelay(); + rackLocalityFullReset = schedConf.getRackLocalityFullReset(); + + // re-init this since max allocation could have changed + this.minimumAllocationFactor = Resources.ratio(resourceCalculator, + Resources.subtract(queueAllocationSettings.getMaximumAllocation(), + queueAllocationSettings.getMinimumAllocation()), + queueAllocationSettings.getMaximumAllocation()); + + StringBuilder aclsString = new StringBuilder(); + for (Map.Entry e : acls.entrySet()) { + aclsString.append(e.getKey() + ":" + e.getValue().getAclString()); + } + + StringBuilder labelStrBuilder = new StringBuilder(); + if (accessibleNodeLabels != null) { + for (String nodeLabel : accessibleNodeLabels) { + labelStrBuilder.append(nodeLabel).append(","); + } + } + + defaultAppPriorityPerQueue = + Priority.newInstance(conf.getDefaultApplicationPriorityConfPerQueue(getQueuePath())); + + // Validate leaf queue's user's weights. + float queueUserLimit = Math.min(100.0f, conf.getUserLimit(getQueuePath())); + getUserWeights().validateForLeafQueue(queueUserLimit, getQueuePath()); + usersManager.updateUserWeights(); + + LOG.info( + "Initializing " + getQueuePath() + "\n" + + getExtendedCapacityOrWeightString() + "\n" + + "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity() + + " [= parentAbsoluteCapacity * capacity ]" + "\n" + + "maxCapacity = " + queueCapacities.getMaximumCapacity() + + " [= configuredMaxCapacity ]" + "\n" + "absoluteMaxCapacity = " + + queueCapacities.getAbsoluteMaximumCapacity() + + " [= 1.0 maximumCapacity undefined, " + + "(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]" + + "\n" + "effectiveMinResource=" + + getEffectiveCapacity(CommonNodeLabelsManager.NO_LABEL) + "\n" + + " , effectiveMaxResource=" + + getEffectiveMaxCapacity(CommonNodeLabelsManager.NO_LABEL) + + "\n" + "userLimit = " + usersManager.getUserLimit() + + " [= configuredUserLimit ]" + "\n" + "userLimitFactor = " + + usersManager.getUserLimitFactor() + + " [= configuredUserLimitFactor ]" + "\n" + "maxApplications = " + + maxApplications + + " [= configuredMaximumSystemApplicationsPerQueue or" + + " (int)(configuredMaximumSystemApplications * absoluteCapacity)]" + + "\n" + "maxApplicationsPerUser = " + maxApplicationsPerUser + + " [= (int)(maxApplications * (userLimit / 100.0f) * " + + "userLimitFactor) ]" + "\n" + + "maxParallelApps = " + getMaxParallelApps() + "\n" + + "usedCapacity = " + + + queueCapacities.getUsedCapacity() + " [= usedResourcesMemory / " + + "(clusterResourceMemory * absoluteCapacity)]" + "\n" + + "absoluteUsedCapacity = " + absoluteUsedCapacity + + " [= usedResourcesMemory / clusterResourceMemory]" + "\n" + + "maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent + + " [= configuredMaximumAMResourcePercent ]" + "\n" + + "minimumAllocationFactor = " + minimumAllocationFactor + + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " + + "maximumAllocationMemory ]" + "\n" + "maximumAllocation = " + + queueAllocationSettings.getMaximumAllocation() + + " [= configuredMaxAllocation ]" + "\n" + + "numContainers = " + usageTracker.getNumContainers() + + " [= currentNumContainers ]" + "\n" + "state = " + getState() + + " [= configuredState ]" + "\n" + "acls = " + aclsString + + " [= configuredAcls ]" + "\n" + + "nodeLocalityDelay = " + nodeLocalityDelay + "\n" + + "rackLocalityAdditionalDelay = " + + rackLocalityAdditionalDelay + "\n" + + "labels=" + labelStrBuilder.toString() + "\n" + + "reservationsContinueLooking = " + + reservationsContinueLooking + "\n" + "preemptionDisabled = " + + getPreemptionDisabled() + "\n" + "defaultAppPriorityPerQueue = " + + defaultAppPriorityPerQueue + "\npriority = " + priority + + "\nmaxLifetime = " + getMaximumApplicationLifetime() + + " seconds" + "\ndefaultLifetime = " + + getDefaultApplicationLifetime() + " seconds"); + } finally { + writeLock.unlock(); + } + } + + private String getDefaultNodeLabelExpressionStr() { + String defaultLabelExpression = queueNodeLabelsSettings.getDefaultLabelExpression(); + return defaultLabelExpression == null ? "" : defaultLabelExpression; + } + + /** + * Used only by tests. + */ + @Private + public float getMinimumAllocationFactor() { + return minimumAllocationFactor; + } + + /** + * Used only by tests. + */ + @Private + public float getMaxAMResourcePerQueuePercent() { + return maxAMResourcePerQueuePercent; + } + + public int getMaxApplications() { + return maxApplications; + } + + public int getMaxApplicationsPerUser() { + return maxApplicationsPerUser; + } + + /** + * + * @return UsersManager instance. + */ + public UsersManager getUsersManager() { + return usersManager; + } + + @Override + public AbstractUsersManager getAbstractUsersManager() { + return usersManager; + } + + @Override + public List getChildQueues() { + return null; + } + + /** + * Set user limit. + * @param userLimit new user limit + */ + @VisibleForTesting + void setUserLimit(float userLimit) { + usersManager.setUserLimit(userLimit); + usersManager.userLimitNeedsRecompute(); + } + + /** + * Set user limit factor. + * @param userLimitFactor new user limit factor + */ + @VisibleForTesting + void setUserLimitFactor(float userLimitFactor) { + usersManager.setUserLimitFactor(userLimitFactor); + usersManager.userLimitNeedsRecompute(); + } + + @Override + public int getNumApplications() { + readLock.lock(); + try { + return getNumPendingApplications() + getNumActiveApplications() + getNumNonRunnableApps(); + } finally { + readLock.unlock(); + } + } + + public int getNumPendingApplications() { + readLock.lock(); + try { + return pendingOrderingPolicy.getNumSchedulableEntities(); + } finally { + readLock.unlock(); + } + } + + public int getNumActiveApplications() { + readLock.lock(); + try { + return orderingPolicy.getNumSchedulableEntities(); + } finally { + readLock.unlock(); + } + } + + @Private + public int getNumPendingApplications(String user) { + readLock.lock(); + try { + User u = getUser(user); + if (null == u) { + return 0; + } + return u.getPendingApplications(); + } finally { + readLock.unlock(); + } + } + + @Private + public int getNumActiveApplications(String user) { + readLock.lock(); + try { + User u = getUser(user); + if (null == u) { + return 0; + } + return u.getActiveApplications(); + } finally { + readLock.unlock(); + } + } + + @Private + public float getUserLimit() { + return usersManager.getUserLimit(); + } + + @Private + public float getUserLimitFactor() { + return usersManager.getUserLimitFactor(); + } + + @Override + public QueueInfo getQueueInfo( + boolean includeChildQueues, boolean recursive) { + QueueInfo queueInfo = getQueueInfo(); + return queueInfo; + } + + @Override + public List + getQueueUserAclInfo(UserGroupInformation user) { + readLock.lock(); + try { + QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance( + QueueUserACLInfo.class); + List operations = new ArrayList<>(); + for (QueueACL operation : QueueACL.values()) { + if (hasAccess(operation, user)) { + operations.add(operation); + } + } + + userAclInfo.setQueueName(getQueuePath()); + userAclInfo.setUserAcls(operations); + return Collections.singletonList(userAclInfo); + } finally { + readLock.unlock(); + } + + } + + public String toString() { + readLock.lock(); + try { + return getQueuePath() + ": " + getCapacityOrWeightString() + + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + + ", " + "usedResources=" + usageTracker.getQueueUsage().getUsed() + ", " + + "usedCapacity=" + getUsedCapacity() + ", " + "absoluteUsedCapacity=" + + getAbsoluteUsedCapacity() + ", " + "numApps=" + getNumApplications() + + ", " + "numContainers=" + getNumContainers() + ", " + + "effectiveMinResource=" + + getEffectiveCapacity(CommonNodeLabelsManager.NO_LABEL) + + " , effectiveMaxResource=" + + getEffectiveMaxCapacity(CommonNodeLabelsManager.NO_LABEL); + } finally { + readLock.unlock(); + } + } + + protected String getExtendedCapacityOrWeightString() { + if (queueCapacities.getWeight() != -1) { + return "weight = " + queueCapacities.getWeight() + + " [= (float) configuredCapacity (with w suffix)] " + "\n" + + "normalizedWeight = " + queueCapacities.getNormalizedWeight() + + " [= (float) configuredCapacity / sum(configuredCapacity of " + + "all queues under the parent)]"; + } else { + return "capacity = " + queueCapacities.getCapacity() + + " [= (float) configuredCapacity / 100 ]"; + } + } + + @VisibleForTesting + public User getUser(String userName) { + return usersManager.getUser(userName); + } + + @VisibleForTesting + public User getOrCreateUser(String userName) { + return usersManager.getUserAndAddIfAbsent(userName); + } + + @Private + public List getPriorityACLs() { + readLock.lock(); + try { + return new ArrayList<>(priorityAcls); + } finally { + readLock.unlock(); + } + } + + protected void reinitialize( + CSQueue newlyParsedQueue, Resource clusterResource, + CapacitySchedulerConfiguration configuration) throws + IOException { + + writeLock.lock(); + try { + // We skip reinitialize for dynamic queues, when this is called, and + // new queue is different from this queue, we will make this queue to be + // static queue. + if (newlyParsedQueue != this) { + this.setDynamicQueue(false); + } + + // Sanity check + if (!(newlyParsedQueue instanceof AbstractLeafQueue) || !newlyParsedQueue.getQueuePath() + .equals(getQueuePath())) { + throw new IOException("Trying to reinitialize " + getQueuePath() + " from " + + newlyParsedQueue.getQueuePath()); + } + + AbstractLeafQueue newlyParsedLeafQueue = (AbstractLeafQueue) newlyParsedQueue; + + // don't allow the maximum allocation to be decreased in size + // since we have already told running AM's the size + Resource oldMax = getMaximumAllocation(); + Resource newMax = newlyParsedLeafQueue.getMaximumAllocation(); + + if (!Resources.fitsIn(oldMax, newMax)) { + throw new IOException("Trying to reinitialize " + getQueuePath() + + " the maximum allocation size can not be decreased!" + + " Current setting: " + oldMax + ", trying to set it to: " + + newMax); + } + + setupQueueConfigs(clusterResource, configuration); + } finally { + writeLock.unlock(); + } + } + + @Override + public void reinitialize( + CSQueue newlyParsedQueue, Resource clusterResource) + throws IOException { + reinitialize(newlyParsedQueue, clusterResource, + csContext.getConfiguration()); + } + + @Override + public void submitApplicationAttempt(FiCaSchedulerApp application, + String userName) { + submitApplicationAttempt(application, userName, false); + } + + @Override + public void submitApplicationAttempt(FiCaSchedulerApp application, + String userName, boolean isMoveApp) { + // Careful! Locking order is important! + writeLock.lock(); + try { + // TODO, should use getUser, use this method just to avoid UT failure + // which is caused by wrong invoking order, will fix UT separately + User user = usersManager.getUserAndAddIfAbsent(userName); + + // Add the attempt to our data-structures + addApplicationAttempt(application, user); + } finally { + writeLock.unlock(); + } + + // We don't want to update metrics for move app + if (!isMoveApp) { + boolean unmanagedAM = application.getAppSchedulingInfo() != null && + application.getAppSchedulingInfo().isUnmanagedAM(); + usageTracker.getMetrics().submitAppAttempt(userName, unmanagedAM); + } + + parent.submitApplicationAttempt(application, userName); + } + + @Override + public void submitApplication(ApplicationId applicationId, String userName, + String queue) throws AccessControlException { + // Careful! Locking order is important! + validateSubmitApplication(applicationId, userName, queue); + + // Signal for expired auto deletion. + updateLastSubmittedTimeStamp(); + + // Inform the parent queue + try { + parent.submitApplication(applicationId, userName, queue); + } catch (AccessControlException ace) { + LOG.info("Failed to submit application to parent-queue: " + + parent.getQueuePath(), ace); + throw ace; + } + + } + + public void validateSubmitApplication(ApplicationId applicationId, + String userName, String queue) throws AccessControlException { + writeLock.lock(); + try { + // Check if the queue is accepting jobs + if (getState() != QueueState.RUNNING) { + String msg = "Queue " + getQueuePath() + + " is STOPPED. Cannot accept submission of application: " + + applicationId; + LOG.info(msg); + throw new AccessControlException(msg); + } + + // Check submission limits for queues + //TODO recalculate max applications because they can depend on capacity + if (getNumApplications() >= getMaxApplications() && !(this instanceof AutoCreatedLeafQueue)) { + String msg = + "Queue " + getQueuePath() + " already has " + getNumApplications() + + " applications," + + " cannot accept submission of application: " + applicationId; + LOG.info(msg); + throw new AccessControlException(msg); + } + + // Check submission limits for the user on this queue + User user = usersManager.getUserAndAddIfAbsent(userName); + //TODO recalculate max applications because they can depend on capacity + if (user.getTotalApplications() >= getMaxApplicationsPerUser() && !(this instanceof AutoCreatedLeafQueue)) { + String msg = "Queue " + getQueuePath() + " already has " + user + .getTotalApplications() + " applications from user " + userName + + " cannot accept submission of application: " + applicationId; + LOG.info(msg); + throw new AccessControlException(msg); + } + } finally { + writeLock.unlock(); + } + + try { + parent.validateSubmitApplication(applicationId, userName, queue); + } catch (AccessControlException ace) { + LOG.info("Failed to submit application to parent-queue: " + + parent.getQueuePath(), ace); + throw ace; + } + } + + public Resource getAMResourceLimit() { + return usageTracker.getQueueUsage().getAMLimit(); + } + + public Resource getAMResourceLimitPerPartition(String nodePartition) { + return usageTracker.getQueueUsage().getAMLimit(nodePartition); + } + + @VisibleForTesting + public Resource calculateAndGetAMResourceLimit() { + return calculateAndGetAMResourceLimitPerPartition( + RMNodeLabelsManager.NO_LABEL); + } + + @VisibleForTesting + public Resource getUserAMResourceLimit() { + return getUserAMResourceLimitPerPartition(RMNodeLabelsManager.NO_LABEL, + null); + } + + public Resource getUserAMResourceLimitPerPartition( + String nodePartition, String userName) { + float userWeight = 1.0f; + if (userName != null && getUser(userName) != null) { + userWeight = getUser(userName).getWeight(); + } + + readLock.lock(); + try { + /* + * The user am resource limit is based on the same approach as the user + * limit (as it should represent a subset of that). This means that it uses + * the absolute queue capacity (per partition) instead of the max and is + * modified by the userlimit and the userlimit factor as is the userlimit + */ + float effectiveUserLimit = Math.max(usersManager.getUserLimit() / 100.0f, + 1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1)); + float preWeightedUserLimit = effectiveUserLimit; + effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f); + + Resource queuePartitionResource = getEffectiveCapacity(nodePartition); + + Resource minimumAllocation = queueAllocationSettings.getMinimumAllocation(); + + Resource userAMLimit = Resources.multiplyAndNormalizeUp( + resourceCalculator, queuePartitionResource, + queueCapacities.getMaxAMResourcePercentage(nodePartition) + * effectiveUserLimit * usersManager.getUserLimitFactor(), + minimumAllocation); + + if (getUserLimitFactor() == -1) { + userAMLimit = Resources.multiplyAndNormalizeUp( + resourceCalculator, queuePartitionResource, + queueCapacities.getMaxAMResourcePercentage(nodePartition), + minimumAllocation); + } + + userAMLimit = + Resources.min(resourceCalculator, lastClusterResource, + userAMLimit, + Resources.clone(getAMResourceLimitPerPartition(nodePartition))); + + Resource preWeighteduserAMLimit = + Resources.multiplyAndNormalizeUp( + resourceCalculator, queuePartitionResource, + queueCapacities.getMaxAMResourcePercentage(nodePartition) + * preWeightedUserLimit * usersManager.getUserLimitFactor(), + minimumAllocation); + + if (getUserLimitFactor() == -1) { + preWeighteduserAMLimit = Resources.multiplyAndNormalizeUp( + resourceCalculator, queuePartitionResource, + queueCapacities.getMaxAMResourcePercentage(nodePartition), + minimumAllocation); + } + + preWeighteduserAMLimit = + Resources.min(resourceCalculator, lastClusterResource, + preWeighteduserAMLimit, + Resources.clone(getAMResourceLimitPerPartition(nodePartition))); + usageTracker.getQueueUsage().setUserAMLimit(nodePartition, preWeighteduserAMLimit); + + LOG.debug("Effective user AM limit for \"{}\":{}. Effective weighted" + + " user AM limit: {}. User weight: {}", userName, + preWeighteduserAMLimit, userAMLimit, userWeight); + return userAMLimit; + } finally { + readLock.unlock(); + } + + } + + public Resource calculateAndGetAMResourceLimitPerPartition(String nodePartition) { + writeLock.lock(); + try { + /* + * For non-labeled partition, get the max value from resources currently + * available to the queue and the absolute resources guaranteed for the + * partition in the queue. For labeled partition, consider only the absolute + * resources guaranteed. Multiply this value (based on labeled/ + * non-labeled), * with per-partition am-resource-percent to get the max am + * resource limit for this queue and partition. + */ + Resource queuePartitionResource = getEffectiveCapacity(nodePartition); + + Resource queueCurrentLimit = Resources.none(); + // For non-labeled partition, we need to consider the current queue + // usage limit. + if (nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) { + synchronized (queueResourceLimitsInfo) { + queueCurrentLimit = queueResourceLimitsInfo.getQueueCurrentLimit(); + } + } + + float amResourcePercent = queueCapacities.getMaxAMResourcePercentage(nodePartition); + + // Current usable resource for this queue and partition is the max of + // queueCurrentLimit and queuePartitionResource. + // If any of the resources available to this queue are less than queue's + // guarantee, use the guarantee as the queuePartitionUsableResource + // because nothing less than the queue's guarantee should be used when + // calculating the AM limit. + Resource queuePartitionUsableResource = + (Resources.fitsIn(resourceCalculator, queuePartitionResource, queueCurrentLimit)) ? + queueCurrentLimit : queuePartitionResource; + + Resource amResouceLimit = + Resources.multiplyAndNormalizeUp(resourceCalculator, queuePartitionUsableResource, + amResourcePercent, queueAllocationSettings.getMinimumAllocation()); + + usageTracker.getMetrics().setAMResouceLimit(nodePartition, amResouceLimit); + usageTracker.getQueueUsage().setAMLimit(nodePartition, amResouceLimit); + LOG.debug("Queue: {}, node label : {}, queue partition resource : {}," + + " queue current limit : {}, queue partition usable resource : {}," + + " amResourceLimit : {}", getQueuePath(), nodePartition, + queuePartitionResource, queueCurrentLimit, + queuePartitionUsableResource, amResouceLimit); + return amResouceLimit; + } finally { + writeLock.unlock(); + } + } + + protected void activateApplications() { + writeLock.lock(); + try { + // limit of allowed resource usage for application masters + Map userAmPartitionLimit = new HashMap(); + + // AM Resource Limit for accessible labels can be pre-calculated. + // This will help in updating AMResourceLimit for all labels when queue + // is initialized for the first time (when no applications are present). + for (String nodePartition : getNodeLabelsForQueue()) { + calculateAndGetAMResourceLimitPerPartition(nodePartition); + } + + for (Iterator fsApp = getPendingAppsOrderingPolicy().getAssignmentIterator( + IteratorSelector.EMPTY_ITERATOR_SELECTOR); fsApp.hasNext(); ) { + FiCaSchedulerApp application = fsApp.next(); + ApplicationId applicationId = application.getApplicationId(); + + // Get the am-node-partition associated with each application + // and calculate max-am resource limit for this partition. + String partitionName = application.getAppAMNodePartitionName(); + + Resource amLimit = getAMResourceLimitPerPartition(partitionName); + // Verify whether we already calculated am-limit for this label. + if (amLimit == null) { + amLimit = calculateAndGetAMResourceLimitPerPartition(partitionName); + } + // Check am resource limit. + Resource amIfStarted = Resources.add(application.getAMResource(partitionName), + usageTracker.getQueueUsage().getAMUsed(partitionName)); + + if (LOG.isDebugEnabled()) { + LOG.debug("application " + application.getId() + " AMResource " + + application.getAMResource(partitionName) + + " maxAMResourcePerQueuePercent " + maxAMResourcePerQueuePercent + + " amLimit " + amLimit + " lastClusterResource " + + lastClusterResource + " amIfStarted " + amIfStarted + + " AM node-partition name " + partitionName); + } + + if (!resourceCalculator.fitsIn(amIfStarted, amLimit)) { + if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(resourceCalculator, + lastClusterResource, usageTracker.getQueueUsage().getAMUsed(partitionName), + Resources.none()))) { + LOG.warn("maximum-am-resource-percent is insufficient to start a" + + " single application in queue, it is likely set too low." + + " skipping enforcement to allow at least one application" + " to start"); + } else { + application.updateAMContainerDiagnostics( + SchedulerApplicationAttempt.AMState.INACTIVATED, + CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED); + LOG.debug( + "Not activating application {} as amIfStarted: {}" + " exceeds amLimit: {}", + applicationId, amIfStarted, amLimit); + continue; + } + } + + // Check user am resource limit + User user = usersManager.getUserAndAddIfAbsent(application.getUser()); + Resource userAMLimit = userAmPartitionLimit.get(partitionName); + + // Verify whether we already calculated user-am-limit for this label. + if (userAMLimit == null) { + userAMLimit = getUserAMResourceLimitPerPartition(partitionName, application.getUser()); + userAmPartitionLimit.put(partitionName, userAMLimit); + } + + Resource userAmIfStarted = Resources.add(application.getAMResource(partitionName), + user.getConsumedAMResources(partitionName)); + + if (!resourceCalculator.fitsIn(userAmIfStarted, userAMLimit)) { + if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(resourceCalculator, + lastClusterResource, usageTracker.getQueueUsage().getAMUsed(partitionName), + Resources.none()))) { + LOG.warn("maximum-am-resource-percent is insufficient to start a" + + " single application in queue for user, it is likely set too" + + " low. skipping enforcement to allow at least one application" + " to start"); + } else { + application.updateAMContainerDiagnostics( + AMState.INACTIVATED, + CSAMContainerLaunchDiagnosticsConstants.USER_AM_RESOURCE_LIMIT_EXCEED); + LOG.debug("Not activating application {} for user: {} as" + + " userAmIfStarted: {} exceeds userAmLimit: {}", + applicationId, user, userAmIfStarted, userAMLimit); + continue; + } + } + user.activateApplication(); + orderingPolicy.addSchedulableEntity(application); + application.updateAMContainerDiagnostics(AMState.ACTIVATED, + null); + + usageTracker.getQueueUsage() + .incAMUsed(partitionName, application.getAMResource(partitionName)); + user.getResourceUsage().incAMUsed(partitionName, application.getAMResource(partitionName)); + user.getResourceUsage().setAMLimit(partitionName, userAMLimit); + usageTracker.getMetrics().incAMUsed(partitionName, application.getUser(), + application.getAMResource(partitionName)); + usageTracker.getMetrics() + .setAMResouceLimitForUser(partitionName, application.getUser(), userAMLimit); + fsApp.remove(); + LOG.info("Application " + applicationId + " from user: " + application + .getUser() + " activated in queue: " + getQueuePath()); + } + } finally { + writeLock.unlock(); + } + } + + private void addApplicationAttempt(FiCaSchedulerApp application, User user) { + writeLock.lock(); + try { + applicationAttemptMap.put(application.getApplicationAttemptId(), application); + + if (application.isRunnable()) { + runnableApps.add(application); + LOG.debug("Adding runnable application: {}", + application.getApplicationAttemptId()); + } else { + nonRunnableApps.add(application); + LOG.info("Application attempt {} is not runnable," + + " parallel limit reached", application.getApplicationAttemptId()); + return; + } + + // Accept + user.submitApplication(); + getPendingAppsOrderingPolicy().addSchedulableEntity(application); + + // Activate applications + if (Resources.greaterThan(resourceCalculator, lastClusterResource, lastClusterResource, + Resources.none())) { + activateApplications(); + } else { + application.updateAMContainerDiagnostics(AMState.INACTIVATED, + CSAMContainerLaunchDiagnosticsConstants.CLUSTER_RESOURCE_EMPTY); + LOG.info("Skipping activateApplications for " + + application.getApplicationAttemptId() + + " since cluster resource is " + Resources.none()); + } + + LOG.info( + "Application added -" + " appId: " + application.getApplicationId() + + " user: " + application.getUser() + "," + " leaf-queue: " + + getQueuePath() + " #user-pending-applications: " + user + .getPendingApplications() + " #user-active-applications: " + user + .getActiveApplications() + " #queue-pending-applications: " + + getNumPendingApplications() + " #queue-active-applications: " + + getNumActiveApplications() + + " #queue-nonrunnable-applications: " + + getNumNonRunnableApps()); + } finally { + writeLock.unlock(); + } + } + + @Override + public void finishApplication(ApplicationId application, String user) { + // Inform the activeUsersManager + usersManager.deactivateApplication(user, application); + + appFinished(); + + // Inform the parent queue + parent.finishApplication(application, user); + } + + @Override + public void finishApplicationAttempt(FiCaSchedulerApp application, String queue) { + // Careful! Locking order is important! + removeApplicationAttempt(application, application.getUser()); + parent.finishApplicationAttempt(application, queue); + } + + private void removeApplicationAttempt(FiCaSchedulerApp application, String userName) { + + writeLock.lock(); + try { + // TODO, should use getUser, use this method just to avoid UT failure + // which is caused by wrong invoking order, will fix UT separately + User user = usersManager.getUserAndAddIfAbsent(userName); + + boolean runnable = runnableApps.remove(application); + if (!runnable) { + // removeNonRunnableApp acquires the write lock again, which is fine + if (!removeNonRunnableApp(application)) { + LOG.error("Given app to remove " + application + + " does not exist in queue " + getQueuePath()); + } + } + + String partitionName = application.getAppAMNodePartitionName(); + boolean wasActive = orderingPolicy.removeSchedulableEntity(application); + if (!wasActive) { + pendingOrderingPolicy.removeSchedulableEntity(application); + } else { + usageTracker.getQueueUsage() + .decAMUsed(partitionName, application.getAMResource(partitionName)); + user.getResourceUsage().decAMUsed(partitionName, application.getAMResource(partitionName)); + usageTracker.getMetrics().decAMUsed(partitionName, application.getUser(), + application.getAMResource(partitionName)); + } + applicationAttemptMap.remove(application.getApplicationAttemptId()); + + user.finishApplication(wasActive); + if (user.getTotalApplications() == 0) { + usersManager.removeUser(application.getUser()); + } + + // Check if we can activate more applications + activateApplications(); + + LOG.info( + "Application removed -" + " appId: " + application.getApplicationId() + + " user: " + application.getUser() + " queue: " + getQueuePath() + + " #user-pending-applications: " + user.getPendingApplications() + + " #user-active-applications: " + user.getActiveApplications() + + " #queue-pending-applications: " + getNumPendingApplications() + + " #queue-active-applications: " + getNumActiveApplications()); + } finally { + writeLock.unlock(); + } + } + + private FiCaSchedulerApp getApplication(ApplicationAttemptId applicationAttemptId) { + return applicationAttemptMap.get(applicationAttemptId); + } + + private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) { + // Set preemption-allowed: + // For leaf queue, only under-utilized queue is allowed to preempt resources from other queues + if (!usageTracker.getQueueResourceQuotas().getEffectiveMinResource(nodePartition) + .equals(Resources.none())) { + limits.setIsAllowPreemption(Resources.lessThan(resourceCalculator, + csContext.getClusterResource(), usageTracker.getQueueUsage().getUsed(nodePartition), + usageTracker.getQueueResourceQuotas().getEffectiveMinResource(nodePartition))); + return; + } + + float usedCapacity = queueCapacities.getAbsoluteUsedCapacity(nodePartition); + float guaranteedCapacity = queueCapacities.getAbsoluteCapacity(nodePartition); + limits.setIsAllowPreemption(usedCapacity < guaranteedCapacity); + } + + private CSAssignment allocateFromReservedContainer(Resource clusterResource, + CandidateNodeSet candidates, ResourceLimits currentResourceLimits, + SchedulingMode schedulingMode) { + + // Irrespective of Single / Multi Node Placement, the allocate from + // Reserved Container has to happen only for the single node which + // CapacityScheduler#allocateFromReservedContainer invokes with. + // Else In Multi Node Placement, there won't be any Allocation or + // Reserve of new containers when there is a RESERVED container on + // a node which is full. + FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates); + if (node != null) { + RMContainer reservedContainer = node.getReservedContainer(); + if (reservedContainer != null) { + FiCaSchedulerApp application = getApplication(reservedContainer.getApplicationAttemptId()); + + if (null != application) { + ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager, node, + SystemClock.getInstance().getTime(), application); + CSAssignment assignment = + application.assignContainers(clusterResource, candidates, currentResourceLimits, + schedulingMode, reservedContainer); + return assignment; + } + } + } + + return null; + } + + private ConcurrentMap getUserLimitCache(String partition, + SchedulingMode schedulingMode) { + synchronized (userLimitsCache) { + long latestVersion = usersManager.getLatestVersionOfUsersState(); + + if (latestVersion != this.currentUserLimitCacheVersion) { + // User limits cache needs invalidating + this.currentUserLimitCacheVersion = latestVersion; + userLimitsCache.clear(); + + Map> uLCByPartition = + new HashMap<>(); + userLimitsCache.put(partition, uLCByPartition); + + ConcurrentMap uLCBySchedulingMode = new ConcurrentHashMap<>(); + uLCByPartition.put(schedulingMode, uLCBySchedulingMode); + + return uLCBySchedulingMode; + } + + // User limits cache does not need invalidating + Map> uLCByPartition = + userLimitsCache.get(partition); + if (uLCByPartition == null) { + uLCByPartition = new HashMap<>(); + userLimitsCache.put(partition, uLCByPartition); + } + + ConcurrentMap uLCBySchedulingMode = + uLCByPartition.get(schedulingMode); + if (uLCBySchedulingMode == null) { + uLCBySchedulingMode = new ConcurrentHashMap<>(); + uLCByPartition.put(schedulingMode, uLCBySchedulingMode); + } + + return uLCBySchedulingMode; + } + } + + @Override + public CSAssignment assignContainers(Resource clusterResource, + CandidateNodeSet candidates, ResourceLimits currentResourceLimits, + SchedulingMode schedulingMode) { + updateCurrentResourceLimits(currentResourceLimits, clusterResource); + FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates); + + if (LOG.isDebugEnabled()) { + LOG.debug("assignContainers: partition=" + candidates.getPartition() + + " #applications=" + orderingPolicy.getNumSchedulableEntities()); + } + + setPreemptionAllowed(currentResourceLimits, candidates.getPartition()); + + // Check for reserved resources, try to allocate reserved container first. + CSAssignment assignment = + allocateFromReservedContainer(clusterResource, candidates, currentResourceLimits, + schedulingMode); + if (null != assignment) { + return assignment; + } + + // if our queue cannot access this node, just return + if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY + && !queueNodeLabelsSettings.isAccessibleToPartition(candidates.getPartition())) { + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, + parent.getQueuePath(), getQueuePath(), ActivityState.REJECTED, + ActivityDiagnosticConstant.QUEUE_NOT_ABLE_TO_ACCESS_PARTITION); + return CSAssignment.NULL_ASSIGNMENT; + } + + // Check if this queue need more resource, simply skip allocation if this + // queue doesn't need more resources. + if (!hasPendingResourceRequest(candidates.getPartition(), clusterResource, + schedulingMode)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skip this queue=" + getQueuePath() + + ", because it doesn't need more resource, schedulingMode=" + + schedulingMode.name() + " node-partition=" + candidates + .getPartition()); + } + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, + parent.getQueuePath(), getQueuePath(), ActivityState.SKIPPED, + ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE); + return CSAssignment.NULL_ASSIGNMENT; + } + + ConcurrentMap userLimits = + this.getUserLimitCache(candidates.getPartition(), schedulingMode); + boolean needAssignToQueueCheck = true; + IteratorSelector sel = new IteratorSelector(); + sel.setPartition(candidates.getPartition()); + for (Iterator assignmentIterator = orderingPolicy.getAssignmentIterator(sel); + assignmentIterator.hasNext(); ) { + FiCaSchedulerApp application = assignmentIterator.next(); + + ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager, node, + SystemClock.getInstance().getTime(), application); + + // Check queue max-capacity limit + Resource appReserved = application.getCurrentReservation(); + if (needAssignToQueueCheck) { + if (!super.canAssignToThisQueue(clusterResource, candidates.getPartition(), + currentResourceLimits, appReserved, schedulingMode)) { + ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(activitiesManager, node, + application, application.getPriority(), + ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, parent.getQueuePath(), + getQueuePath(), ActivityState.REJECTED, + ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); + return CSAssignment.NULL_ASSIGNMENT; + } + // If there was no reservation and canAssignToThisQueue returned + // true, there is no reason to check further. + if (!this.reservationsContinueLooking || appReserved.equals(Resources.none())) { + needAssignToQueueCheck = false; + } + } + + CachedUserLimit cul = userLimits.get(application.getUser()); + Resource cachedUserLimit = null; + if (cul != null) { + cachedUserLimit = cul.userLimit; + } + Resource userLimit = + computeUserLimitAndSetHeadroom(application, clusterResource, candidates.getPartition(), + schedulingMode, cachedUserLimit); + if (cul == null) { + cul = new CachedUserLimit(userLimit); + CachedUserLimit retVal = userLimits.putIfAbsent(application.getUser(), cul); + if (retVal != null) { + // another thread updated the user limit cache before us + cul = retVal; + userLimit = cul.userLimit; + } + } + // Check user limit + boolean userAssignable = true; + if (!cul.canAssign && Resources.fitsIn(appReserved, cul.reservation)) { + userAssignable = false; + } else { + userAssignable = + canAssignToUser(clusterResource, application.getUser(), userLimit, application, + candidates.getPartition(), currentResourceLimits); + if (!userAssignable && Resources.fitsIn(cul.reservation, appReserved)) { + cul.canAssign = false; + cul.reservation = appReserved; + } + } + if (!userAssignable) { + application.updateAMContainerDiagnostics(AMState.ACTIVATED, + "User capacity has reached its maximum limit."); + ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(activitiesManager, node, + application, application.getPriority(), + ActivityDiagnosticConstant.QUEUE_HIT_USER_MAX_CAPACITY_LIMIT); + continue; + } + + // Try to schedule + assignment = application.assignContainers(clusterResource, candidates, currentResourceLimits, + schedulingMode, null); + + if (LOG.isDebugEnabled()) { + LOG.debug( + "post-assignContainers for application " + application.getApplicationId()); + application.showRequests(); + } + + // Did we schedule or reserve a container? + Resource assigned = assignment.getResource(); + + if (Resources.greaterThan(resourceCalculator, clusterResource, assigned, Resources.none())) { + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, parent.getQueuePath(), + getQueuePath(), ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); + return assignment; + } else if (assignment.getSkippedType() == CSAssignment.SkippedType.OTHER) { + ActivitiesLogger.APP.finishSkippedAppAllocationRecording(activitiesManager, + application.getApplicationId(), ActivityState.SKIPPED, + ActivityDiagnosticConstant.EMPTY); + application.updateNodeInfoForAMDiagnostics(node); + } else if (assignment.getSkippedType() == CSAssignment.SkippedType.QUEUE_LIMIT) { + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, parent.getQueuePath(), + getQueuePath(), ActivityState.REJECTED, + () -> ActivityDiagnosticConstant.QUEUE_DO_NOT_HAVE_ENOUGH_HEADROOM + " from " + + application.getApplicationId()); + return assignment; + } else { + // If we don't allocate anything, and it is not skipped by application, + // we will return to respect FIFO of applications + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, parent.getQueuePath(), + getQueuePath(), ActivityState.SKIPPED, + ActivityDiagnosticConstant.QUEUE_SKIPPED_TO_RESPECT_FIFO); + ActivitiesLogger.APP.finishSkippedAppAllocationRecording(activitiesManager, + application.getApplicationId(), ActivityState.SKIPPED, + ActivityDiagnosticConstant.EMPTY); + return CSAssignment.NULL_ASSIGNMENT; + } + } + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, parent.getQueuePath(), + getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY); + + return CSAssignment.NULL_ASSIGNMENT; + } + + @Override + public boolean accept(Resource cluster, + ResourceCommitRequest request) { + ContainerAllocationProposal allocation = + request.getFirstAllocatedOrReservedContainer(); + SchedulerContainer schedulerContainer = + allocation.getAllocatedOrReservedContainer(); + + // Do not check limits when allocation from a reserved container + if (allocation.getAllocateFromReservedContainer() == null) { + readLock.lock(); + try { + FiCaSchedulerApp app = schedulerContainer.getSchedulerApplicationAttempt(); + String username = app.getUser(); + String p = schedulerContainer.getNodePartition(); + + // check user-limit + Resource userLimit = + computeUserLimitAndSetHeadroom(app, cluster, p, allocation.getSchedulingMode(), null); + + // Deduct resources that we can release + User user = getUser(username); + if (user == null) { + LOG.debug("User {} has been removed!", username); + return false; + } + Resource usedResource = Resources.clone(user.getUsed(p)); + Resources.subtractFrom(usedResource, request.getTotalReleasedResource()); + + if (Resources.greaterThan(resourceCalculator, cluster, usedResource, userLimit)) { + LOG.debug("Used resource={} exceeded user-limit={}", usedResource, + userLimit); + return false; + } + } finally { + readLock.unlock(); + } + } + + return super.accept(cluster, request); + } + + private void internalReleaseContainer(Resource clusterResource, + SchedulerContainer schedulerContainer) { + RMContainer rmContainer = schedulerContainer.getRmContainer(); + + AbstractLeafQueue targetLeafQueue = + schedulerContainer.getSchedulerApplicationAttempt().getCSLeafQueue(); + + if (targetLeafQueue == this) { + // When trying to preempt containers from the same queue + if (rmContainer.getState() == RMContainerState.RESERVED) { + // For other reserved containers + // This is a reservation exchange, complete previous reserved container + completedContainer(clusterResource, schedulerContainer.getSchedulerApplicationAttempt(), + schedulerContainer.getSchedulerNode(), rmContainer, + SchedulerUtils.createAbnormalContainerStatus(rmContainer.getContainerId(), + SchedulerUtils.UNRESERVED_CONTAINER), RMContainerEventType.RELEASED, null, false); + } + } else { + // When trying to preempt containers from different queue -- this + // is for lazy preemption feature (kill preemption candidate in scheduling + // cycle). + targetLeafQueue.completedContainer(clusterResource, + schedulerContainer.getSchedulerApplicationAttempt(), + schedulerContainer.getSchedulerNode(), schedulerContainer.getRmContainer(), + SchedulerUtils.createPreemptedContainerStatus(rmContainer.getContainerId(), + SchedulerUtils.PREEMPTED_CONTAINER), RMContainerEventType.KILL, null, false); + } + } + + private void releaseContainers(Resource clusterResource, + ResourceCommitRequest request) { + for (SchedulerContainer c : request.getContainersToRelease()) { + internalReleaseContainer(clusterResource, c); + } + + // Handle container reservation looking, or lazy preemption case: + if (null != request.getContainersToAllocate() && !request.getContainersToAllocate().isEmpty()) { + for (ContainerAllocationProposal context : request.getContainersToAllocate()) { + if (null != context.getToRelease()) { + for (SchedulerContainer c : context.getToRelease()) { + internalReleaseContainer(clusterResource, c); + } + } + } + } + } + + public void apply(Resource cluster, + ResourceCommitRequest request) { + // Do we need to call parent queue's apply? + boolean applyToParentQueue = false; + + releaseContainers(cluster, request); + + writeLock.lock(); + try { + if (request.anythingAllocatedOrReserved()) { + ContainerAllocationProposal allocation = + request.getFirstAllocatedOrReservedContainer(); + SchedulerContainer schedulerContainer = + allocation.getAllocatedOrReservedContainer(); + + // Do not modify queue when allocation from reserved container + if (allocation.getAllocateFromReservedContainer() == null) { + // Only invoke apply() of ParentQueue when new allocation / + // reservation happen. + applyToParentQueue = true; + // Book-keeping + // Note: Update headroom to account for current allocation too... + allocateResource(cluster, schedulerContainer.getSchedulerApplicationAttempt(), + allocation.getAllocatedOrReservedResource(), schedulerContainer.getNodePartition(), + schedulerContainer.getRmContainer()); + orderingPolicy.containerAllocated(schedulerContainer.getSchedulerApplicationAttempt(), + schedulerContainer.getRmContainer()); + } + + // Update reserved resource + if (Resources.greaterThan(resourceCalculator, cluster, request.getTotalReservedResource(), + Resources.none())) { + incReservedResource(schedulerContainer.getNodePartition(), + request.getTotalReservedResource()); + } + } + } finally { + writeLock.unlock(); + } + + if (parent != null && applyToParentQueue) { + parent.apply(cluster, request); + } + } + + protected Resource getHeadroom(User user, Resource queueCurrentLimit, + Resource clusterResource, FiCaSchedulerApp application) { + return getHeadroom(user, queueCurrentLimit, clusterResource, application, + RMNodeLabelsManager.NO_LABEL); + } + + protected Resource getHeadroom(User user, Resource queueCurrentLimit, + Resource clusterResource, FiCaSchedulerApp application, + String partition) { + return getHeadroom(user, queueCurrentLimit, clusterResource, + getResourceLimitForActiveUsers(application.getUser(), clusterResource, partition, + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), partition); + } + + private Resource getHeadroom(User user, + Resource currentPartitionResourceLimit, Resource clusterResource, + Resource userLimitResource, String partition) { + /** + * Headroom is: + * min( + * min(userLimit, queueMaxCap) - userConsumed, + * queueMaxCap - queueUsedResources + * ) + * + * ( which can be expressed as, + * min (userLimit - userConsumed, queuMaxCap - userConsumed, + * queueMaxCap - queueUsedResources) + * ) + * + * given that queueUsedResources >= userConsumed, this simplifies to + * + * >> min (userlimit - userConsumed, queueMaxCap - queueUsedResources) << + * + * sum of queue max capacities of multiple queue's will be greater than the + * actual capacity of a given partition, hence we need to ensure that the + * headroom is not greater than the available resource for a given partition + * + * headroom = min (unused resourcelimit of a label, calculated headroom ) + */ + currentPartitionResourceLimit = + partition.equals(RMNodeLabelsManager.NO_LABEL) ? currentPartitionResourceLimit : + getQueueMaxResource(partition); + + Resource headroom = Resources.componentwiseMin( + Resources.subtractNonNegative(userLimitResource, user.getUsed(partition)), + Resources.subtractNonNegative(currentPartitionResourceLimit, + usageTracker.getQueueUsage().getUsed(partition))); + // Normalize it before return + headroom = Resources.roundDown(resourceCalculator, headroom, + queueAllocationSettings.getMinimumAllocation()); + + //headroom = min (unused resourcelimit of a label, calculated headroom ) + Resource clusterPartitionResource = labelManager.getResourceByLabel(partition, clusterResource); + Resource clusterFreePartitionResource = Resources.subtract(clusterPartitionResource, + csContext.getClusterResourceUsage().getUsed(partition)); + headroom = + Resources.min(resourceCalculator, clusterPartitionResource, clusterFreePartitionResource, + headroom); + return headroom; + } + + private void setQueueResourceLimitsInfo(Resource clusterResource) { + synchronized (queueResourceLimitsInfo) { + queueResourceLimitsInfo.setQueueCurrentLimit(cachedResourceLimitsForHeadroom.getLimit()); + queueResourceLimitsInfo.setClusterResource(clusterResource); + } + } + + // It doesn't necessarily to hold application's lock here. + @Lock({AbstractLeafQueue.class}) + Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application, + Resource clusterResource, String nodePartition, SchedulingMode schedulingMode, + Resource userLimit) { + String user = application.getUser(); + User queueUser = getUser(user); + if (queueUser == null) { + LOG.debug("User {} has been removed!", user); + return Resources.none(); + } + + // Compute user limit respect requested labels, + // TODO, need consider headroom respect labels also + if (userLimit == null) { + userLimit = + getResourceLimitForActiveUsers(application.getUser(), clusterResource, nodePartition, + schedulingMode); + } + setQueueResourceLimitsInfo(clusterResource); + + Resource headroom = usageTracker.getMetrics().getUserMetrics(user) == null ? Resources.none() : + getHeadroom(queueUser, cachedResourceLimitsForHeadroom.getLimit(), clusterResource, + userLimit, nodePartition); + + if (LOG.isDebugEnabled()) { + LOG.debug( + "Headroom calculation for user " + user + ": " + " userLimit=" + userLimit + + " queueMaxAvailRes=" + cachedResourceLimitsForHeadroom.getLimit() + " consumed=" + + queueUser.getUsed() + " partition=" + nodePartition); + } + + CapacityHeadroomProvider headroomProvider = + new CapacityHeadroomProvider(queueUser, this, application, queueResourceLimitsInfo); + + application.setHeadroomProvider(headroomProvider); + + usageTracker.getMetrics().setAvailableResourcesToUser(nodePartition, user, headroom); + + return userLimit; + } + + @Lock(NoLock.class) + public int getNodeLocalityDelay() { + return nodeLocalityDelay; + } + + @Lock(NoLock.class) + public int getRackLocalityAdditionalDelay() { + return rackLocalityAdditionalDelay; + } + + @Lock(NoLock.class) + public boolean getRackLocalityFullReset() { + return rackLocalityFullReset; + } + + /** + * + * @param userName + * Name of user who has submitted one/more app to given queue. + * @param clusterResource + * total cluster resource + * @param nodePartition + * partition name + * @param schedulingMode + * scheduling mode + * RESPECT_PARTITION_EXCLUSIVITY/IGNORE_PARTITION_EXCLUSIVITY + * @return Computed User Limit + */ + public Resource getResourceLimitForActiveUsers(String userName, Resource clusterResource, + String nodePartition, SchedulingMode schedulingMode) { + return usersManager.getComputedResourceLimitForActiveUsers(userName, clusterResource, + nodePartition, schedulingMode); + } + + /** + * + * @param userName + * Name of user who has submitted one/more app to given queue. + * @param clusterResource + * total cluster resource + * @param nodePartition + * partition name + * @param schedulingMode + * scheduling mode + * RESPECT_PARTITION_EXCLUSIVITY/IGNORE_PARTITION_EXCLUSIVITY + * @return Computed User Limit + */ + public Resource getResourceLimitForAllUsers(String userName, Resource clusterResource, + String nodePartition, SchedulingMode schedulingMode) { + return usersManager.getComputedResourceLimitForAllUsers(userName, clusterResource, + nodePartition, schedulingMode); + } + + @Private + protected boolean canAssignToUser(Resource clusterResource, + String userName, Resource limit, FiCaSchedulerApp application, + String nodePartition, ResourceLimits currentResourceLimits) { + + readLock.lock(); + try { + User user = getUser(userName); + if (user == null) { + LOG.debug("User {} has been removed!", userName); + return false; + } + + currentResourceLimits.setAmountNeededUnreserve(Resources.none()); + + // Note: We aren't considering the current request since there is a fixed + // overhead of the AM, but it's a > check, not a >= check, so... + if (Resources.greaterThan(resourceCalculator, clusterResource, user.getUsed(nodePartition), + limit)) { + // if enabled, check to see if could we potentially use this node instead + // of a reserved node if the application has reserved containers + if (this.reservationsContinueLooking) { + if (Resources.lessThanOrEqual(resourceCalculator, clusterResource, + Resources.subtract(user.getUsed(), application.getCurrentReservation()), limit)) { + + if (LOG.isDebugEnabled()) { + LOG.debug("User " + userName + " in queue " + getQueuePath() + + " will exceed limit based on reservations - " + " consumed: " + user.getUsed() + + " reserved: " + application.getCurrentReservation() + " limit: " + limit); + } + Resource amountNeededToUnreserve = + Resources.subtract(user.getUsed(nodePartition), limit); + // we can only acquire a new container if we unreserve first to + // respect user-limit + currentResourceLimits.setAmountNeededUnreserve(amountNeededToUnreserve); + return true; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("User " + userName + " in queue " + getQueuePath() + + " will exceed limit - " + " consumed: " + user + .getUsed(nodePartition) + " limit: " + limit); + } + return false; + } + return true; + } finally { + readLock.unlock(); + } + } + + @Override + protected void setDynamicQueueProperties(CapacitySchedulerConfiguration configuration) { + // set to -1, to disable it + configuration.setUserLimitFactor(getQueuePath(), -1); + // Set Max AM percentage to a higher value + configuration.setMaximumApplicationMasterResourcePerQueuePercent( + getQueuePath(), 1f); + super.setDynamicQueueProperties(configuration); + } + + private void updateSchedulerHealthForCompletedContainer(RMContainer rmContainer, + ContainerStatus containerStatus) { + // Update SchedulerHealth for released / preempted container + SchedulerHealth schedulerHealth = csContext.getSchedulerHealth(); + if (null == schedulerHealth) { + // Only do update if we have schedulerHealth + return; + } + + if (containerStatus.getExitStatus() == ContainerExitStatus.PREEMPTED) { + schedulerHealth.updatePreemption(Time.now(), rmContainer.getAllocatedNode(), + rmContainer.getContainerId(), getQueuePath()); + schedulerHealth.updateSchedulerPreemptionCounts(1); + } else { + schedulerHealth.updateRelease(csContext.getLastNodeUpdateTime(), + rmContainer.getAllocatedNode(), rmContainer.getContainerId(), getQueuePath()); + } + } + + /** + * Recalculate QueueUsage Ratio. + * + * @param clusterResource + * Total Cluster Resource + * @param nodePartition + * Partition + */ + public void recalculateQueueUsageRatio(Resource clusterResource, String nodePartition) { + writeLock.lock(); + try { + ResourceUsage queueResourceUsage = getQueueResourceUsage(); + + if (nodePartition == null) { + for (String partition : Sets.union(getQueueCapacities().getNodePartitionsSet(), + queueResourceUsage.getNodePartitionsSet())) { + usersManager.updateUsageRatio(partition, clusterResource); + } + } else { + usersManager.updateUsageRatio(nodePartition, clusterResource); + } + } finally { + writeLock.unlock(); + } + } + + @Override + public void completedContainer(Resource clusterResource, FiCaSchedulerApp application, + FiCaSchedulerNode node, RMContainer rmContainer, ContainerStatus containerStatus, + RMContainerEventType event, CSQueue childQueue, boolean sortQueues) { + // Update SchedulerHealth for released / preempted container + updateSchedulerHealthForCompletedContainer(rmContainer, containerStatus); + + if (application != null) { + boolean removed = false; + + // Careful! Locking order is important! + writeLock.lock(); + try { + Container container = rmContainer.getContainer(); + + // Inform the application & the node + // Note: It's safe to assume that all state changes to RMContainer + // happen under scheduler's lock... + // So, this is, in effect, a transaction across application & node + if (rmContainer.getState() == RMContainerState.RESERVED) { + removed = application.unreserve(rmContainer.getReservedSchedulerKey(), node, rmContainer); + } else { + removed = application.containerCompleted(rmContainer, containerStatus, event, + node.getPartition()); + + node.releaseContainer(rmContainer.getContainerId(), false); + } + + // Book-keeping + if (removed) { + + // Inform the ordering policy + orderingPolicy.containerReleased(application, rmContainer); + + releaseResource(clusterResource, application, container.getResource(), + node.getPartition(), rmContainer); + } + } finally { + writeLock.unlock(); + } + + if (removed) { + // Inform the parent queue _outside_ of the leaf-queue lock + parent.completedContainer(clusterResource, application, node, rmContainer, null, event, + this, sortQueues); + } + } + + // Notify PreemptionManager + csContext.getPreemptionManager().removeKillableContainer( + new KillableContainer(rmContainer, node.getPartition(), getQueuePath())); + + // Update preemption metrics if exit status is PREEMPTED + if (containerStatus != null + && ContainerExitStatus.PREEMPTED == containerStatus.getExitStatus()) { + updateQueuePreemptionMetrics(rmContainer); + } + } + + void allocateResource(Resource clusterResource, SchedulerApplicationAttempt application, + Resource resource, String nodePartition, RMContainer rmContainer) { + writeLock.lock(); + try { + super.allocateResource(clusterResource, resource, nodePartition); + + // handle ignore exclusivity container + if (null != rmContainer && rmContainer.getNodeLabelExpression() + .equals(RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals( + RMNodeLabelsManager.NO_LABEL)) { + TreeSet rmContainers = null; + if (null == (rmContainers = ignorePartitionExclusivityRMContainers.get(nodePartition))) { + rmContainers = new TreeSet<>(); + ignorePartitionExclusivityRMContainers.put(nodePartition, rmContainers); + } + rmContainers.add(rmContainer); + } + + // Update user metrics + String userName = application.getUser(); + + // Increment user's resource usage. + User user = usersManager.updateUserResourceUsage(userName, resource, + nodePartition, true); + + Resource partitionHeadroom = Resources.createResource(0, 0); + if (usageTracker.getMetrics().getUserMetrics(userName) != null) { + partitionHeadroom = + getHeadroom(user, cachedResourceLimitsForHeadroom.getLimit(), clusterResource, + getResourceLimitForActiveUsers(userName, clusterResource, nodePartition, + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodePartition); + } + usageTracker.getMetrics() + .setAvailableResourcesToUser(nodePartition, userName, partitionHeadroom); + + if (LOG.isDebugEnabled()) { + LOG.debug(getQueuePath() + " user=" + userName + " used=" + + usageTracker.getQueueUsage().getUsed(nodePartition) + " numContainers=" + + usageTracker.getNumContainers() + " headroom = " + application.getHeadroom() + + " user-resources=" + user.getUsed()); + } + } finally { + writeLock.unlock(); + } + } + + void releaseResource(Resource clusterResource, FiCaSchedulerApp application, Resource resource, + String nodePartition, RMContainer rmContainer) { + writeLock.lock(); + try { + super.releaseResource(clusterResource, resource, nodePartition); + + // handle ignore exclusivity container + if (null != rmContainer && rmContainer.getNodeLabelExpression() + .equals(RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals( + RMNodeLabelsManager.NO_LABEL)) { + if (ignorePartitionExclusivityRMContainers.containsKey(nodePartition)) { + Set rmContainers = ignorePartitionExclusivityRMContainers.get(nodePartition); + rmContainers.remove(rmContainer); + if (rmContainers.isEmpty()) { + ignorePartitionExclusivityRMContainers.remove(nodePartition); + } + } + } + + // Update user metrics + String userName = application.getUser(); + User user = usersManager.updateUserResourceUsage(userName, resource, + nodePartition, false); + + Resource partitionHeadroom = Resources.createResource(0, 0); + if (usageTracker.getMetrics().getUserMetrics(userName) != null) { + partitionHeadroom = + getHeadroom(user, cachedResourceLimitsForHeadroom.getLimit(), clusterResource, + getResourceLimitForActiveUsers(userName, clusterResource, nodePartition, + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodePartition); + } + usageTracker.getMetrics() + .setAvailableResourcesToUser(nodePartition, userName, partitionHeadroom); + + if (LOG.isDebugEnabled()) { + LOG.debug( + getQueuePath() + " used=" + usageTracker.getQueueUsage().getUsed() + " numContainers=" + + usageTracker.getNumContainers() + " user=" + userName + " user-resources=" + + user.getUsed()); + } + } finally { + writeLock.unlock(); + } + } + + private void updateCurrentResourceLimits(ResourceLimits currentResourceLimits, + Resource clusterResource) { + // TODO: need consider non-empty node labels when resource limits supports + // node labels + // Even if ParentQueue will set limits respect child's max queue capacity, + // but when allocating reserved container, CapacityScheduler doesn't do + // this. So need cap limits by queue's max capacity here. + this.cachedResourceLimitsForHeadroom = new ResourceLimits(currentResourceLimits.getLimit()); + Resource queueMaxResource = getEffectiveMaxCapacityDown(RMNodeLabelsManager.NO_LABEL, + queueAllocationSettings.getMinimumAllocation()); + this.cachedResourceLimitsForHeadroom.setLimit( + Resources.min(resourceCalculator, clusterResource, queueMaxResource, + currentResourceLimits.getLimit())); + } + + @Override + public void updateClusterResource(Resource clusterResource, + ResourceLimits currentResourceLimits) { + writeLock.lock(); + try { + lastClusterResource = clusterResource; + + updateAbsoluteCapacities(); + + super.updateEffectiveResources(clusterResource); + + // Update maximum applications for the queue and for users + updateMaximumApplications(csContext.getConfiguration()); + + updateCurrentResourceLimits(currentResourceLimits, clusterResource); + + // Update headroom info based on new cluster resource value + // absoluteMaxCapacity now, will be replaced with absoluteMaxAvailCapacity + // during allocation + setQueueResourceLimitsInfo(clusterResource); + + // Update user consumedRatios + recalculateQueueUsageRatio(clusterResource, null); + + // Update metrics + CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, this, labelManager, + null); + // Update configured capacity/max-capacity for default partition only + CSQueueUtils.updateConfiguredCapacityMetrics(resourceCalculator, + labelManager.getResourceByLabel(null, clusterResource), RMNodeLabelsManager.NO_LABEL, + this); + + // queue metrics are updated, more resource may be available + // activate the pending applications if possible + activateApplications(); + + // In case of any resource change, invalidate recalculateULCount to clear + // the computed user-limit. + usersManager.userLimitNeedsRecompute(); + + // Update application properties + for (FiCaSchedulerApp application : orderingPolicy.getSchedulableEntities()) { + computeUserLimitAndSetHeadroom(application, clusterResource, RMNodeLabelsManager.NO_LABEL, + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, null); + } + } finally { + writeLock.unlock(); + } + } + + @Override + public void incUsedResource(String nodeLabel, Resource resourceToInc, + SchedulerApplicationAttempt application) { + usersManager.updateUserResourceUsage(application.getUser(), resourceToInc, nodeLabel, true); + super.incUsedResource(nodeLabel, resourceToInc, application); + } + + @Override + public void decUsedResource(String nodeLabel, Resource resourceToDec, + SchedulerApplicationAttempt application) { + usersManager.updateUserResourceUsage(application.getUser(), resourceToDec, nodeLabel, false); + super.decUsedResource(nodeLabel, resourceToDec, application); + } + + public void incAMUsedResource(String nodeLabel, Resource resourceToInc, + SchedulerApplicationAttempt application) { + User user = getUser(application.getUser()); + if (user == null) { + return; + } + + user.getResourceUsage().incAMUsed(nodeLabel, resourceToInc); + // ResourceUsage has its own lock, no addition lock needs here. + usageTracker.getQueueUsage().incAMUsed(nodeLabel, resourceToInc); + } + + public void decAMUsedResource(String nodeLabel, Resource resourceToDec, + SchedulerApplicationAttempt application) { + User user = getUser(application.getUser()); + if (user == null) { + return; + } + + user.getResourceUsage().decAMUsed(nodeLabel, resourceToDec); + // ResourceUsage has its own lock, no addition lock needs here. + usageTracker.getQueueUsage().decAMUsed(nodeLabel, resourceToDec); + } + + @Override + public void recoverContainer(Resource clusterResource, + SchedulerApplicationAttempt attempt, RMContainer rmContainer) { + if (rmContainer.getState().equals(RMContainerState.COMPLETED)) { + return; + } + if (rmContainer.getExecutionType() != ExecutionType.GUARANTEED) { + return; + } + // Careful! Locking order is important! + writeLock.lock(); + try { + FiCaSchedulerNode node = csContext.getNode(rmContainer.getContainer().getNodeId()); + allocateResource(clusterResource, attempt, rmContainer.getContainer().getResource(), + node.getPartition(), rmContainer); + } finally { + writeLock.unlock(); + } + + parent.recoverContainer(clusterResource, attempt, rmContainer); + } + + /** + * Obtain (read-only) collection of pending applications. + */ + public Collection getPendingApplications() { + return Collections.unmodifiableCollection(pendingOrderingPolicy.getSchedulableEntities()); + } + + /** + * Obtain (read-only) collection of active applications. + */ + public Collection getApplications() { + return Collections.unmodifiableCollection(orderingPolicy.getSchedulableEntities()); + } + + /** + * Obtain (read-only) collection of all applications. + */ + public Collection getAllApplications() { + Collection apps = + new HashSet(pendingOrderingPolicy.getSchedulableEntities()); + apps.addAll(orderingPolicy.getSchedulableEntities()); + + return Collections.unmodifiableCollection(apps); + } + + /** + * Get total pending resource considering user limit for the leaf queue. This + * will be used for calculating pending resources in the preemption monitor. + * + * Consider the headroom for each user in the queue. + * Total pending for the queue = + * sum(for each user(min((user's headroom), sum(user's pending requests)))) + * NOTE: + * @param clusterResources clusterResource + * @param partition node partition + * @param deductReservedFromPending When a container is reserved in CS, + * pending resource will not be deducted. + * This could lead to double accounting when + * doing preemption: + * In normal cases, we should deduct reserved + * resource from pending to avoid + * excessive preemption. + * @return Total pending resource considering user limit + */ + public Resource getTotalPendingResourcesConsideringUserLimit( + Resource clusterResources, String partition, + boolean deductReservedFromPending) { + readLock.lock(); + try { + Map userNameToHeadroom = + new HashMap<>(); + Resource totalPendingConsideringUserLimit = Resource.newInstance(0, 0); + for (FiCaSchedulerApp app : getApplications()) { + String userName = app.getUser(); + if (!userNameToHeadroom.containsKey(userName)) { + User user = getUsersManager().getUserAndAddIfAbsent(userName); + Resource headroom = Resources.subtract( + getResourceLimitForActiveUsers(app.getUser(), clusterResources, + partition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), + user.getUsed(partition)); + // Make sure headroom is not negative. + headroom = Resources.componentwiseMax(headroom, Resources.none()); + userNameToHeadroom.put(userName, headroom); + } + + // Check if we need to deduct reserved from pending + Resource pending = app.getAppAttemptResourceUsage().getPending( + partition); + if (deductReservedFromPending) { + pending = Resources.subtract(pending, + app.getAppAttemptResourceUsage().getReserved(partition)); + } + pending = Resources.componentwiseMax(pending, Resources.none()); + + Resource minpendingConsideringUserLimit = Resources.componentwiseMin( + userNameToHeadroom.get(userName), pending); + Resources.addTo(totalPendingConsideringUserLimit, + minpendingConsideringUserLimit); + Resources.subtractFrom(userNameToHeadroom.get(userName), + minpendingConsideringUserLimit); + } + return totalPendingConsideringUserLimit; + } finally { + readLock.unlock(); + } + + } + + @Override + public void collectSchedulerApplications( + Collection apps) { + readLock.lock(); + try { + for (FiCaSchedulerApp pendingApp : pendingOrderingPolicy + .getSchedulableEntities()) { + apps.add(pendingApp.getApplicationAttemptId()); + } + for (FiCaSchedulerApp app : orderingPolicy.getSchedulableEntities()) { + apps.add(app.getApplicationAttemptId()); + } + } finally { + readLock.unlock(); + } + + } + + @Override + public void attachContainer(Resource clusterResource, + FiCaSchedulerApp application, RMContainer rmContainer) { + if (application != null && rmContainer != null + && rmContainer.getExecutionType() == ExecutionType.GUARANTEED) { + FiCaSchedulerNode node = + csContext.getNode(rmContainer.getContainer().getNodeId()); + allocateResource(clusterResource, application, rmContainer.getContainer() + .getResource(), node.getPartition(), rmContainer); + LOG.info("movedContainer" + " container=" + rmContainer.getContainer() + + " containerState="+ rmContainer.getState() + + " resource=" + rmContainer.getContainer().getResource() + + " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity() + + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + + usageTracker.getQueueUsage().getUsed() + " cluster=" + clusterResource); + // Inform the parent queue + parent.attachContainer(clusterResource, application, rmContainer); + } + } + + @Override + public void detachContainer(Resource clusterResource, + FiCaSchedulerApp application, RMContainer rmContainer) { + if (application != null && rmContainer != null + && rmContainer.getExecutionType() == ExecutionType.GUARANTEED) { + FiCaSchedulerNode node = + csContext.getNode(rmContainer.getContainer().getNodeId()); + releaseResource(clusterResource, application, rmContainer.getContainer() + .getResource(), node.getPartition(), rmContainer); + LOG.info("movedContainer" + " container=" + rmContainer.getContainer() + + " containerState="+ rmContainer.getState() + + " resource=" + rmContainer.getContainer().getResource() + + " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity() + + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + + usageTracker.getQueueUsage().getUsed() + " cluster=" + clusterResource); + // Inform the parent queue + parent.detachContainer(clusterResource, application, rmContainer); + } + } + + /** + * @return all ignored partition exclusivity RMContainers in the LeafQueue, + * this will be used by preemption policy. + */ + public Map> + getIgnoreExclusivityRMContainers() { + Map> clonedMap = new HashMap<>(); + + readLock.lock(); + try { + for (Map.Entry> entry : ignorePartitionExclusivityRMContainers + .entrySet()) { + clonedMap.put(entry.getKey(), new TreeSet<>(entry.getValue())); + } + + return clonedMap; + + } finally { + readLock.unlock(); + } + } + + public void setCapacity(float capacity) { + queueCapacities.setCapacity(capacity); + } + + public void setCapacity(String nodeLabel, float capacity) { + queueCapacities.setCapacity(nodeLabel, capacity); + } + + public void setAbsoluteCapacity(float absoluteCapacity) { + queueCapacities.setAbsoluteCapacity(absoluteCapacity); + } + + public void setAbsoluteCapacity(String nodeLabel, float absoluteCapacity) { + queueCapacities.setAbsoluteCapacity(nodeLabel, absoluteCapacity); + } + + public void setMaxApplicationsPerUser(int maxApplicationsPerUser) { + this.maxApplicationsPerUser = maxApplicationsPerUser; + } + + public void setMaxApplications(int maxApplications) { + this.maxApplications = maxApplications; + } + + public void setMaxAMResourcePerQueuePercent( + float maxAMResourcePerQueuePercent) { + this.maxAMResourcePerQueuePercent = maxAMResourcePerQueuePercent; + } + + public OrderingPolicy + getOrderingPolicy() { + return orderingPolicy; + } + + void setOrderingPolicy( + OrderingPolicy orderingPolicy) { + writeLock.lock(); + try { + if (null != this.orderingPolicy) { + orderingPolicy.addAllSchedulableEntities( + this.orderingPolicy.getSchedulableEntities()); + } + this.orderingPolicy = orderingPolicy; + } finally { + writeLock.unlock(); + } + } + + @Override + public Priority getDefaultApplicationPriority() { + return defaultAppPriorityPerQueue; + } + + public void updateApplicationPriority(SchedulerApplication app, + Priority newAppPriority) { + writeLock.lock(); + try { + FiCaSchedulerApp attempt = app.getCurrentAppAttempt(); + boolean isActive = orderingPolicy.removeSchedulableEntity(attempt); + if (!isActive) { + pendingOrderingPolicy.removeSchedulableEntity(attempt); + } + // Update new priority in SchedulerApplication + attempt.setPriority(newAppPriority); + + if (isActive) { + orderingPolicy.addSchedulableEntity(attempt); + } else { + pendingOrderingPolicy.addSchedulableEntity(attempt); + } + } finally { + writeLock.unlock(); + } + } + + public OrderingPolicy + getPendingAppsOrderingPolicy() { + return pendingOrderingPolicy; + } + + /* + * Holds shared values used by all applications in + * the queue to calculate headroom on demand + */ + static class QueueResourceLimitsInfo { + private Resource queueCurrentLimit; + private Resource clusterResource; + + public void setQueueCurrentLimit(Resource currentLimit) { + this.queueCurrentLimit = currentLimit; + } + + public Resource getQueueCurrentLimit() { + return queueCurrentLimit; + } + + public void setClusterResource(Resource clusterResource) { + this.clusterResource = clusterResource; + } + + public Resource getClusterResource() { + return clusterResource; + } + } + + @Override + public void stopQueue() { + writeLock.lock(); + try { + if (getNumApplications() > 0) { + updateQueueState(QueueState.DRAINING); + } else { + updateQueueState(QueueState.STOPPED); + } + } finally { + writeLock.unlock(); + } + } + + void updateMaximumApplications(CapacitySchedulerConfiguration conf) { + int maxAppsForQueue = conf.getMaximumApplicationsPerQueue(getQueuePath()); + + int maxDefaultPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue(); + int maxSystemApps = conf.getMaximumSystemApplications(); + int baseMaxApplications = maxDefaultPerQueueApps > 0 ? + Math.min(maxDefaultPerQueueApps, maxSystemApps) + : maxSystemApps; + + String maxLabel = RMNodeLabelsManager.NO_LABEL; + if (maxAppsForQueue < 0) { + if (maxDefaultPerQueueApps > 0 && this.capacityConfigType + != CapacityConfigType.ABSOLUTE_RESOURCE) { + maxAppsForQueue = baseMaxApplications; + } else { + for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { + int maxApplicationsByLabel = (int) (baseMaxApplications + * queueCapacities.getAbsoluteCapacity(label)); + if (maxApplicationsByLabel > maxAppsForQueue) { + maxAppsForQueue = maxApplicationsByLabel; + maxLabel = label; + } + } + } + } + + setMaxApplications(maxAppsForQueue); + + updateMaxAppsPerUser(); + + LOG.info("LeafQueue:" + getQueuePath() + + "update max app related, maxApplications=" + + maxAppsForQueue + ", maxApplicationsPerUser=" + + maxApplicationsPerUser + ", Abs Cap:" + queueCapacities + .getAbsoluteCapacity(maxLabel) + ", Cap: " + queueCapacities + .getCapacity(maxLabel) + ", MaxCap : " + queueCapacities + .getMaximumCapacity(maxLabel)); + } + + private void updateMaxAppsPerUser() { + int maxAppsPerUser = maxApplications; + if (getUsersManager().getUserLimitFactor() != -1) { + int maxApplicationsWithUserLimits = (int) (maxApplications + * (getUsersManager().getUserLimit() / 100.0f) + * getUsersManager().getUserLimitFactor()); + maxAppsPerUser = Math.min(maxApplications, + maxApplicationsWithUserLimits); + } + + setMaxApplicationsPerUser(maxAppsPerUser); + } + + /** + * Get all valid users in this queue. + * @return user list + */ + public Set getAllUsers() { + return this.getUsersManager().getUsers().keySet(); + } + + static class CachedUserLimit { + final Resource userLimit; + volatile boolean canAssign = true; + volatile Resource reservation = Resources.none(); + + CachedUserLimit(Resource userLimit) { + this.userLimit = userLimit; + } + } + + private void updateQueuePreemptionMetrics(RMContainer rmc) { + final long usedMillis = rmc.getFinishTime() - rmc.getCreationTime(); + final long usedSeconds = usedMillis / DateUtils.MILLIS_PER_SECOND; + CSQueueMetrics metrics = usageTracker.getMetrics(); + Resource containerResource = rmc.getAllocatedResource(); + metrics.preemptContainer(); + long mbSeconds = (containerResource.getMemorySize() * usedMillis) + / DateUtils.MILLIS_PER_SECOND; + long vcSeconds = (containerResource.getVirtualCores() * usedMillis) + / DateUtils.MILLIS_PER_SECOND; + metrics.updatePreemptedMemoryMBSeconds(mbSeconds); + metrics.updatePreemptedVcoreSeconds(vcSeconds); + metrics.updatePreemptedResources(containerResource); + metrics.updatePreemptedSecondsForCustomResources(containerResource, + usedSeconds); + metrics.updatePreemptedForCustomResources(containerResource); + } + + @Override + int getNumRunnableApps() { + readLock.lock(); + try { + return runnableApps.size(); + } finally { + readLock.unlock(); + } + } + + int getNumNonRunnableApps() { + readLock.lock(); + try { + return nonRunnableApps.size(); + } finally { + readLock.unlock(); + } + } + + boolean removeNonRunnableApp(FiCaSchedulerApp app) { + writeLock.lock(); + try { + return nonRunnableApps.remove(app); + } finally { + writeLock.unlock(); + } + } + + List getCopyOfNonRunnableAppSchedulables() { + List appsToReturn = new ArrayList<>(); + readLock.lock(); + try { + appsToReturn.addAll(nonRunnableApps); + } finally { + readLock.unlock(); + } + return appsToReturn; + } + + @Override + public boolean isEligibleForAutoDeletion() { + return isDynamicQueue() && getNumApplications() == 0 + && csContext.getConfiguration(). + isAutoExpiredDeletionEnabled(this.getQueuePath()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java index 7311be77baaa19..57050b193abd68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java @@ -36,15 +36,19 @@ * ManagedParentQueue for auto created dynamic queues */ public class AutoCreatedLeafQueue extends AbstractAutoCreatedLeafQueue { - private static final Logger LOG = LoggerFactory .getLogger(AutoCreatedLeafQueue.class); public AutoCreatedLeafQueue(CapacitySchedulerContext cs, String queueName, ManagedParentQueue parent) throws IOException { + // TODO once YARN-10907 is merged the duplicated collection of + // leafQueueConfigs won't be necessary super(cs, parent.getLeafQueueConfigs(queueName), queueName, parent, null); + super.setupQueueConfigs(cs.getClusterResource(), parent.getLeafQueueConfigs(queueName)); + + LOG.debug("Initialized AutoCreatedLeafQueue: name={}, fullname={}", queueName, getQueuePath()); updateCapacitiesToZero(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSMaxRunningAppsEnforcer.java index 93d001773138f7..fedde057a9010c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSMaxRunningAppsEnforcer.java @@ -182,7 +182,7 @@ public void updateRunnabilityOnAppRemoval(FiCaSchedulerApp app) { // the queue was already at its max before the removal. // Thus we find the ancestor queue highest in the tree for which the app // that was at its maxRunningApps before the removal. - LeafQueue queue = app.getCSLeafQueue(); + AbstractLeafQueue queue = app.getCSLeafQueue(); AbstractCSQueue highestQueueWithAppsNowRunnable = (queue.getNumRunnableApps() == queue.getMaxParallelApps() - 1) ? queue : null; @@ -243,7 +243,7 @@ private void updateAppsRunnability(List> } if (checkRunnabilityWithUpdate(next)) { - LeafQueue nextQueue = next.getCSLeafQueue(); + AbstractLeafQueue nextQueue = next.getCSLeafQueue(); LOG.info("{} is now runnable in {}", next.getApplicationAttemptId(), nextQueue); trackRunnableApp(next); @@ -322,9 +322,9 @@ private void untrackNonRunnableApp(FiCaSchedulerApp app) { private void gatherPossiblyRunnableAppLists(AbstractCSQueue queue, List> appLists) { if (queue.getNumRunnableApps() < queue.getMaxParallelApps()) { - if (queue instanceof LeafQueue) { + if (queue instanceof AbstractLeafQueue) { appLists.add( - ((LeafQueue)queue).getCopyOfNonRunnableAppSchedulables()); + ((AbstractLeafQueue)queue).getCopyOfNonRunnableAppSchedulables()); } else { for (CSQueue child : queue.getChildQueues()) { gatherPossiblyRunnableAppLists((AbstractCSQueue) child, appLists); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java index 140a2acdbc7582..de31cd1361c252 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java @@ -27,13 +27,13 @@ public class CapacityHeadroomProvider { UsersManager.User user; - LeafQueue queue; + AbstractLeafQueue queue; FiCaSchedulerApp application; - LeafQueue.QueueResourceLimitsInfo queueResourceLimitsInfo; + AbstractLeafQueue.QueueResourceLimitsInfo queueResourceLimitsInfo; - public CapacityHeadroomProvider(UsersManager.User user, LeafQueue queue, + public CapacityHeadroomProvider(UsersManager.User user, AbstractLeafQueue queue, FiCaSchedulerApp application, - LeafQueue.QueueResourceLimitsInfo queueResourceLimitsInfo) { + AbstractLeafQueue.QueueResourceLimitsInfo queueResourceLimitsInfo) { this.user = user; this.queue = queue; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index bd1089bd6b66ec..befb82a70e22e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -917,7 +917,7 @@ private void addApplicationOnRecovery(ApplicationId applicationId, throw new QueueInvalidException(queueErrorMsg); } } - if (!(queue instanceof LeafQueue)) { + if (!(queue instanceof AbstractLeafQueue)) { // During RM restart, this means leaf queue was converted to a parent // queue, which is not supported for running apps. if (!appShouldFailFast) { @@ -942,7 +942,7 @@ private void addApplicationOnRecovery(ApplicationId applicationId, // that means its previous state was DRAINING. So we auto transit // the state to DRAINING for recovery. if (queue.getState() == QueueState.STOPPED) { - ((LeafQueue) queue).recoverDrainingState(); + ((AbstractLeafQueue) queue).recoverDrainingState(); } // Submit to the queue try { @@ -1090,7 +1090,7 @@ private void addApplication(ApplicationId applicationId, String queueName, return; } - if (!(queue instanceof LeafQueue)) { + if (!(queue instanceof AbstractLeafQueue)) { String message = "Application " + applicationId + " submitted by user : " + user + " to non-leaf queue : " + queueName; @@ -1242,7 +1242,7 @@ private void doneApplication(ApplicationId applicationId, return; } CSQueue queue = (CSQueue) application.getQueue(); - if (!(queue instanceof LeafQueue)) { + if (!(queue instanceof AbstractLeafQueue)) { LOG.error("Cannot finish application " + "from non-leaf queue: " + queue .getQueuePath()); } else{ @@ -1301,7 +1301,7 @@ private void doneApplicationAttempt( // Inform the queue Queue queue = attempt.getQueue(); CSQueue csQueue = (CSQueue) queue; - if (!(csQueue instanceof LeafQueue)) { + if (!(csQueue instanceof AbstractLeafQueue)) { LOG.error( "Cannot finish application " + "from non-leaf queue: " + csQueue.getQueuePath()); @@ -1367,7 +1367,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, // Release containers releaseContainers(release, application); - LeafQueue updateDemandForQueue = null; + AbstractLeafQueue updateDemandForQueue = null; // Sanity check for new allocation requests normalizeResourceRequests(ask); @@ -1398,7 +1398,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, // Update application requests if (application.updateResourceRequests(ask) || application .updateSchedulingRequests(schedulingRequests)) { - updateDemandForQueue = (LeafQueue) application.getQueue(); + updateDemandForQueue = (AbstractLeafQueue) application.getQueue(); } if (LOG.isDebugEnabled()) { @@ -1783,7 +1783,7 @@ private void allocateFromReservedContainer(FiCaSchedulerNode node, LOG.debug("Trying to fulfill reservation for application {} on node: {}", reservedApplication.getApplicationId(), node.getNodeID()); - LeafQueue queue = ((LeafQueue) reservedApplication.getQueue()); + AbstractLeafQueue queue = ((AbstractLeafQueue) reservedApplication.getQueue()); CSAssignment assignment = queue.assignContainers(getClusterResource(), new SimpleCandidateNodeSet<>(node), // TODO, now we only consider limits for parent for non-labeled @@ -2386,7 +2386,7 @@ protected void completedContainerInternal( } // Inform the queue - LeafQueue queue = (LeafQueue) application.getQueue(); + AbstractLeafQueue queue = (AbstractLeafQueue) application.getQueue(); queue.completedContainer(getClusterResource(), application, node, rmContainer, containerStatus, event, null, true); } @@ -2673,7 +2673,7 @@ public void setEntitlement(String inQueue, QueueEntitlement entitlement) throws YarnException { writeLock.lock(); try { - LeafQueue queue = this.queueManager.getAndCheckLeafQueue(inQueue); + AbstractLeafQueue queue = this.queueManager.getAndCheckLeafQueue(inQueue); AbstractManagedParentQueue parent = (AbstractManagedParentQueue) queue.getParent(); @@ -2716,10 +2716,10 @@ public String moveApplication(ApplicationId appId, throw new YarnException("App to be moved " + appId + " not found."); } String sourceQueueName = application.getQueue().getQueueName(); - LeafQueue source = + AbstractLeafQueue source = this.queueManager.getAndCheckLeafQueue(sourceQueueName); String destQueueName = handleMoveToPlanQueue(targetQueueName); - LeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName); + AbstractLeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName); String user = application.getUser(); try { @@ -2777,7 +2777,7 @@ public void preValidateMoveApplication(ApplicationId appId, ((CSQueue) queue).getQueuePath() : queue.getQueueName(); this.queueManager.getAndCheckLeafQueue(sourceQueueName); String destQueueName = handleMoveToPlanQueue(newQueue); - LeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName); + AbstractLeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName); // Validation check - ACLs, submission limits for user & queue String user = application.getUser(); // Check active partition only when attempt is available @@ -2804,7 +2804,7 @@ public void preValidateMoveApplication(ApplicationId appId, * @param dest * @throws YarnException */ - private void checkQueuePartition(FiCaSchedulerApp app, LeafQueue dest) + private void checkQueuePartition(FiCaSchedulerApp app, AbstractLeafQueue dest) throws YarnException { if (!YarnConfiguration.areNodeLabelsEnabled(conf)) { return; @@ -2854,7 +2854,7 @@ public Resource getMaximumResourceCapability(String queueName) { } return getMaximumResourceCapability(); } - if (!(queue instanceof LeafQueue)) { + if (!(queue instanceof AbstractLeafQueue)) { LOG.error("queue " + queueName + " is not an leaf queue"); return getMaximumResourceCapability(); } @@ -2863,7 +2863,7 @@ public Resource getMaximumResourceCapability(String queueName) { // getMaximumResourceCapability() returns maximum allocation considers // per-node maximum resources. So return (component-wise) min of the two. - Resource queueMaxAllocation = ((LeafQueue)queue).getMaximumAllocation(); + Resource queueMaxAllocation = queue.getMaximumAllocation(); Resource clusterMaxAllocationConsiderNodeMax = getMaximumResourceCapability(); @@ -2989,7 +2989,7 @@ public Priority updateApplicationPriority(Priority newPriority, // As we use iterator over a TreeSet for OrderingPolicy, once we change // priority then reinsert back to make order correct. - LeafQueue queue = (LeafQueue) getQueue(rmApp.getQueue()); + AbstractLeafQueue queue = (AbstractLeafQueue) getQueue(rmApp.getQueue()); queue.updateApplicationPriority(application, appPriority); LOG.info("Priority '" + appPriority + "' is updated in queue :" @@ -3404,14 +3404,14 @@ public long checkAndGetApplicationLifetime(String queueName, readLock.lock(); try { CSQueue queue = getQueue(queueName); - if (queue == null || !(queue instanceof LeafQueue)) { + if (!(queue instanceof AbstractLeafQueue)) { return lifetimeRequestedByApp; } long defaultApplicationLifetime = - ((LeafQueue) queue).getDefaultApplicationLifetime(); + queue.getDefaultApplicationLifetime(); long maximumApplicationLifetime = - ((LeafQueue) queue).getMaximumApplicationLifetime(); + queue.getMaximumApplicationLifetime(); // check only for maximum, that's enough because default can't // exceed maximum @@ -3434,7 +3434,7 @@ public long checkAndGetApplicationLifetime(String queueName, @Override public long getMaximumApplicationLifetime(String queueName) { CSQueue queue = getQueue(queueName); - if (queue == null || !(queue instanceof LeafQueue)) { + if (!(queue instanceof AbstractLeafQueue)) { if (isAmbiguous(queueName)) { LOG.error("Ambiguous queue reference: " + queueName + " please use full queue path instead."); @@ -3444,7 +3444,7 @@ public long getMaximumApplicationLifetime(String queueName) { return -1; } // In seconds - return ((LeafQueue) queue).getMaximumApplicationLifetime(); + return queue.getMaximumApplicationLifetime(); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java index fd601ac58170e6..147f392ad9280c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java @@ -186,7 +186,7 @@ private static void validateParentQueueConversion(CSQueue oldQueue, + " is set to true"); } - if (newQueue instanceof LeafQueue) { + if (newQueue instanceof AbstractLeafQueue) { LOG.info("Converting the parent queue: {} to leaf queue.", oldQueue.getQueuePath()); } } @@ -194,7 +194,7 @@ private static void validateParentQueueConversion(CSQueue oldQueue, private static void validateLeafQueueConversion(CSQueue oldQueue, CSQueue newQueue) throws IOException { - if (oldQueue instanceof LeafQueue && newQueue instanceof ParentQueue) { + if (oldQueue instanceof AbstractLeafQueue && newQueue instanceof ParentQueue) { if (isEitherQueueStopped(oldQueue.getState(), newQueue.getState())) { LOG.info("Converting the leaf queue: {} to parent queue.", oldQueue.getQueuePath()); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index f0c8a27f954933..407383d3bc0fcf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -375,8 +375,8 @@ public static void setQueueAcls(YarnAuthorizationProvider authorizer, permissions.add( new Permission(csQueue.getPrivilegedEntity(), csQueue.getACLs())); - if (queue instanceof LeafQueue) { - LeafQueue lQueue = (LeafQueue) queue; + if (queue instanceof AbstractLeafQueue) { + AbstractLeafQueue lQueue = (AbstractLeafQueue) queue; // Clear Priority ACLs first since reinitialize also call same. appPriorityACLManager.clearPriorityACLs(lQueue.getQueuePath()); @@ -397,17 +397,17 @@ public static void setQueueAcls(YarnAuthorizationProvider authorizer, * @throws YarnException if the queue does not exist or the queue * is not the type of LeafQueue. */ - public LeafQueue getAndCheckLeafQueue(String queue) throws YarnException { + public AbstractLeafQueue getAndCheckLeafQueue(String queue) throws YarnException { CSQueue ret = this.getQueue(queue); if (ret == null) { throw new YarnException("The specified Queue: " + queue + " doesn't exist"); } - if (!(ret instanceof LeafQueue)) { + if (!(ret instanceof AbstractLeafQueue)) { throw new YarnException("The specified Queue: " + queue + " is not a Leaf Queue."); } - return (LeafQueue) ret; + return (AbstractLeafQueue) ret; } /** @@ -527,7 +527,7 @@ public void addLegacyDynamicQueue(Queue queue) * @throws YarnException if the given path is not eligible to be auto created * @throws IOException if the given path can not be added to the parent */ - public LeafQueue createQueue(QueuePath queue) + public AbstractLeafQueue createQueue(QueuePath queue) throws YarnException, IOException { String leafQueueName = queue.getLeafName(); String parentQueueName = queue.getParent(); @@ -668,7 +668,7 @@ private LeafQueue createAutoQueue(QueuePath queue) return leafQueue; } - private LeafQueue createLegacyAutoQueue(QueuePath queue) + private AbstractLeafQueue createLegacyAutoQueue(QueuePath queue) throws IOException, SchedulerDynamicEditException { CSQueue parentQueue = getQueue(queue.getParent()); // Case 1: Handle ManagedParentQueue diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 4592f2afd198fa..b9fa932f14141d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -19,128 +19,18 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import java.io.IOException; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.DateUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.security.AccessControlException; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.util.Sets; -import org.apache.hadoop.util.Time; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerExitStatus; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ExecutionType; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.QueueACL; -import org.apache.hadoop.yarn.api.records.QueueInfo; -import org.apache.hadoop.yarn.api.records.QueueState; -import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; -import org.apache.hadoop.yarn.security.AccessType; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesLogger; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerAllocationProposal; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyForPendingApps; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.IteratorSelector; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy; -import org.apache.hadoop.yarn.server.utils.Lock; -import org.apache.hadoop.yarn.server.utils.Lock.NoLock; -import org.apache.hadoop.yarn.util.SystemClock; -import org.apache.hadoop.yarn.util.resource.Resources; - -import org.apache.hadoop.classification.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Private @Unstable -public class LeafQueue extends AbstractCSQueue { +public class LeafQueue extends AbstractLeafQueue { private static final Logger LOG = LoggerFactory.getLogger(LeafQueue.class); - private float absoluteUsedCapacity = 0.0f; - - // TODO the max applications should consider label - protected int maxApplications; - protected volatile int maxApplicationsPerUser; - - private float maxAMResourcePerQueuePercent; - - private volatile int nodeLocalityDelay; - private volatile int rackLocalityAdditionalDelay; - private volatile boolean rackLocalityFullReset; - - Map applicationAttemptMap = - new ConcurrentHashMap<>(); - - private Priority defaultAppPriorityPerQueue; - - private final OrderingPolicy pendingOrderingPolicy; - - private volatile float minimumAllocationFactor; - - private final RecordFactory recordFactory = - RecordFactoryProvider.getRecordFactory(null); - - private final UsersManager usersManager; - - // cache last cluster resource to compute actual capacity - private Resource lastClusterResource = Resources.none(); - - private final QueueResourceLimitsInfo queueResourceLimitsInfo = - new QueueResourceLimitsInfo(); - - private volatile ResourceLimits cachedResourceLimitsForHeadroom = null; - - private volatile OrderingPolicy orderingPolicy = null; - - // Map>> - // Not thread safe: only the last level is a ConcurrentMap - @VisibleForTesting - Map>> - userLimitsCache = new HashMap<>(); - - // Not thread safe - @VisibleForTesting - long currentUserLimitCacheVersion = 0; - - // record all ignore partition exclusivityRMContainer, this will be used to do - // preemption, key is the partition of the RMContainer allocated on - private Map> ignorePartitionExclusivityRMContainers = - new ConcurrentHashMap<>(); - - List priorityAcls = - new ArrayList(); - - private final List runnableApps = new ArrayList<>(); - private final List nonRunnableApps = new ArrayList<>(); - @SuppressWarnings({ "unchecked", "rawtypes" }) public LeafQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { @@ -157,2330 +47,10 @@ public LeafQueue(CapacitySchedulerContext cs, CapacitySchedulerConfiguration configuration, String queueName, CSQueue parent, CSQueue old, boolean isDynamic) throws IOException { - super(cs, configuration, queueName, parent, old); - setDynamicQueue(isDynamic); - - this.usersManager = new UsersManager(usageTracker.getMetrics(), this, labelManager, csContext, - resourceCalculator); - - // One time initialization is enough since it is static ordering policy - this.pendingOrderingPolicy = new FifoOrderingPolicyForPendingApps(); - - LOG.debug("LeafQueue: name={}, fullname={}", queueName, getQueuePath()); + super(cs, configuration, queueName, parent, old, isDynamic); setupQueueConfigs(cs.getClusterResource(), configuration); - } - - @SuppressWarnings("checkstyle:nowhitespaceafter") - protected void setupQueueConfigs(Resource clusterResource, - CapacitySchedulerConfiguration conf) throws - IOException { - writeLock.lock(); - try { - CapacitySchedulerConfiguration schedConf = csContext.getConfiguration(); - super.setupQueueConfigs(clusterResource, conf); - - this.lastClusterResource = clusterResource; - - this.cachedResourceLimitsForHeadroom = new ResourceLimits( - clusterResource); - - // Initialize headroom info, also used for calculating application - // master resource limits. Since this happens during queue initialization - // and all queues may not be realized yet, we'll use (optimistic) - // absoluteMaxCapacity (it will be replaced with the more accurate - // absoluteMaxAvailCapacity during headroom/userlimit/allocation events) - setQueueResourceLimitsInfo(clusterResource); - - setOrderingPolicy( - conf.getAppOrderingPolicy(getQueuePath())); - - usersManager.setUserLimit(conf.getUserLimit(getQueuePath())); - usersManager.setUserLimitFactor(conf.getUserLimitFactor(getQueuePath())); - - maxAMResourcePerQueuePercent = - conf.getMaximumApplicationMasterResourcePerQueuePercent( - getQueuePath()); - - maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath()); - if (maxApplications < 0) { - int maxGlobalPerQueueApps = - csContext.getConfiguration().getGlobalMaximumApplicationsPerQueue(); - if (maxGlobalPerQueueApps > 0) { - maxApplications = maxGlobalPerQueueApps; - } - } - - priorityAcls = conf.getPriorityAcls(getQueuePath(), - csContext.getMaxClusterLevelAppPriority()); - - Set accessibleNodeLabels = this.queueNodeLabelsSettings.getAccessibleNodeLabels(); - if (!SchedulerUtils.checkQueueLabelExpression(accessibleNodeLabels, - this.queueNodeLabelsSettings.getDefaultLabelExpression(), null)) { - throw new IOException( - "Invalid default label expression of " + " queue=" + getQueuePath() - + " doesn't have permission to access all labels " - + "in default label expression. labelExpression of resource request=" - + getDefaultNodeLabelExpressionStr() + ". Queue labels=" + ( - getAccessibleNodeLabels() == null ? - "" : - StringUtils - .join(getAccessibleNodeLabels().iterator(), ','))); - } - - nodeLocalityDelay = schedConf.getNodeLocalityDelay(); - rackLocalityAdditionalDelay = schedConf - .getRackLocalityAdditionalDelay(); - rackLocalityFullReset = schedConf - .getRackLocalityFullReset(); - - // re-init this since max allocation could have changed - this.minimumAllocationFactor = Resources.ratio(resourceCalculator, - Resources.subtract( - queueAllocationSettings.getMaximumAllocation(), - queueAllocationSettings.getMinimumAllocation()), - queueAllocationSettings.getMaximumAllocation()); - - StringBuilder aclsString = new StringBuilder(); - for (Map.Entry e : acls.entrySet()) { - aclsString.append(e.getKey() + ":" + e.getValue().getAclString()); - } - - StringBuilder labelStrBuilder = new StringBuilder(); - if (accessibleNodeLabels != null) { - for (String nodeLabel : accessibleNodeLabels) { - labelStrBuilder.append(nodeLabel).append(","); - } - } - - defaultAppPriorityPerQueue = Priority.newInstance( - conf.getDefaultApplicationPriorityConfPerQueue(getQueuePath())); - - // Validate leaf queue's user's weights. - float queueUserLimit = Math.min(100.0f, conf.getUserLimit(getQueuePath())); - getUserWeights().validateForLeafQueue(queueUserLimit, getQueuePath()); - usersManager.updateUserWeights(); - - LOG.info( - "Initializing " + getQueuePath() + "\n" + - getExtendedCapacityOrWeightString() + "\n" - + "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity() - + " [= parentAbsoluteCapacity * capacity ]" + "\n" - + "maxCapacity = " + queueCapacities.getMaximumCapacity() - + " [= configuredMaxCapacity ]" + "\n" + "absoluteMaxCapacity = " - + queueCapacities.getAbsoluteMaximumCapacity() - + " [= 1.0 maximumCapacity undefined, " - + "(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]" - + "\n" + "effectiveMinResource=" + - getEffectiveCapacity(CommonNodeLabelsManager.NO_LABEL) + "\n" - + " , effectiveMaxResource=" + - getEffectiveMaxCapacity(CommonNodeLabelsManager.NO_LABEL) - + "\n" + "userLimit = " + usersManager.getUserLimit() - + " [= configuredUserLimit ]" + "\n" + "userLimitFactor = " - + usersManager.getUserLimitFactor() - + " [= configuredUserLimitFactor ]" + "\n" + "maxApplications = " - + maxApplications - + " [= configuredMaximumSystemApplicationsPerQueue or" - + " (int)(configuredMaximumSystemApplications * absoluteCapacity)]" - + "\n" + "maxApplicationsPerUser = " + maxApplicationsPerUser - + " [= (int)(maxApplications * (userLimit / 100.0f) * " - + "userLimitFactor) ]" + "\n" - + "maxParallelApps = " + getMaxParallelApps() + "\n" - + "usedCapacity = " + - + queueCapacities.getUsedCapacity() + " [= usedResourcesMemory / " - + "(clusterResourceMemory * absoluteCapacity)]" + "\n" - + "absoluteUsedCapacity = " + absoluteUsedCapacity - + " [= usedResourcesMemory / clusterResourceMemory]" + "\n" - + "maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent - + " [= configuredMaximumAMResourcePercent ]" + "\n" - + "minimumAllocationFactor = " + minimumAllocationFactor - + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " - + "maximumAllocationMemory ]" + "\n" + "maximumAllocation = " - + queueAllocationSettings.getMaximumAllocation() + - " [= configuredMaxAllocation ]" + "\n" - + "numContainers = " + usageTracker.getNumContainers() - + " [= currentNumContainers ]" + "\n" + "state = " + getState() - + " [= configuredState ]" + "\n" + "acls = " + aclsString - + " [= configuredAcls ]" + "\n" - + "nodeLocalityDelay = " + nodeLocalityDelay + "\n" - + "rackLocalityAdditionalDelay = " - + rackLocalityAdditionalDelay + "\n" - + "labels=" + labelStrBuilder.toString() + "\n" - + "reservationsContinueLooking = " - + reservationsContinueLooking + "\n" + "preemptionDisabled = " - + getPreemptionDisabled() + "\n" + "defaultAppPriorityPerQueue = " - + defaultAppPriorityPerQueue + "\npriority = " + priority - + "\nmaxLifetime = " + getMaximumApplicationLifetime() - + " seconds" + "\ndefaultLifetime = " - + getDefaultApplicationLifetime() + " seconds"); - } finally { - writeLock.unlock(); - } - } - - private String getDefaultNodeLabelExpressionStr() { - String defaultLabelExpression = queueNodeLabelsSettings.getDefaultLabelExpression(); - return defaultLabelExpression == null ? "" : defaultLabelExpression; - } - - /** - * Used only by tests. - */ - @Private - public float getMinimumAllocationFactor() { - return minimumAllocationFactor; - } - - /** - * Used only by tests. - */ - @Private - public float getMaxAMResourcePerQueuePercent() { - return maxAMResourcePerQueuePercent; - } - - public int getMaxApplications() { - return maxApplications; - } - - public int getMaxApplicationsPerUser() { - return maxApplicationsPerUser; - } - - /** - * - * @return UsersManager instance. - */ - public UsersManager getUsersManager() { - return usersManager; - } - - @Override - public AbstractUsersManager getAbstractUsersManager() { - return usersManager; - } - - @Override - public List getChildQueues() { - return null; - } - - /** - * Set user limit. - * @param userLimit new user limit - */ - @VisibleForTesting - void setUserLimit(float userLimit) { - usersManager.setUserLimit(userLimit); - usersManager.userLimitNeedsRecompute(); - } - - /** - * Set user limit factor. - * @param userLimitFactor new user limit factor - */ - @VisibleForTesting - void setUserLimitFactor(float userLimitFactor) { - usersManager.setUserLimitFactor(userLimitFactor); - usersManager.userLimitNeedsRecompute(); - } - - @Override - public int getNumApplications() { - readLock.lock(); - try { - return getNumPendingApplications() + getNumActiveApplications() + - getNumNonRunnableApps(); - } finally { - readLock.unlock(); - } - } - - public int getNumPendingApplications() { - readLock.lock(); - try { - return pendingOrderingPolicy.getNumSchedulableEntities(); - } finally { - readLock.unlock(); - } - } - - public int getNumActiveApplications() { - readLock.lock(); - try { - return orderingPolicy.getNumSchedulableEntities(); - } finally { - readLock.unlock(); - } - } - - @Private - public int getNumPendingApplications(String user) { - readLock.lock(); - try { - User u = getUser(user); - if (null == u) { - return 0; - } - return u.getPendingApplications(); - } finally { - readLock.unlock(); - } - } - - @Private - public int getNumActiveApplications(String user) { - readLock.lock(); - try { - User u = getUser(user); - if (null == u) { - return 0; - } - return u.getActiveApplications(); - } finally { - readLock.unlock(); - } - } - - @Private - public float getUserLimit() { - return usersManager.getUserLimit(); - } - - @Private - public float getUserLimitFactor() { - return usersManager.getUserLimitFactor(); - } - - @Override - public QueueInfo getQueueInfo( - boolean includeChildQueues, boolean recursive) { - QueueInfo queueInfo = getQueueInfo(); - return queueInfo; - } - - @Override - public List - getQueueUserAclInfo(UserGroupInformation user) { - readLock.lock(); - try { - QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance( - QueueUserACLInfo.class); - List operations = new ArrayList<>(); - for (QueueACL operation : QueueACL.values()) { - if (hasAccess(operation, user)) { - operations.add(operation); - } - } - - userAclInfo.setQueueName(getQueuePath()); - userAclInfo.setUserAcls(operations); - return Collections.singletonList(userAclInfo); - } finally { - readLock.unlock(); - } - - } - - public String toString() { - readLock.lock(); - try { - return getQueuePath() + ": " + getCapacityOrWeightString() - + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() - + ", " + "usedResources=" + usageTracker.getQueueUsage().getUsed() + ", " - + "usedCapacity=" + getUsedCapacity() + ", " + "absoluteUsedCapacity=" - + getAbsoluteUsedCapacity() + ", " + "numApps=" + getNumApplications() - + ", " + "numContainers=" + getNumContainers() + ", " - + "effectiveMinResource=" + - getEffectiveCapacity(CommonNodeLabelsManager.NO_LABEL) + - " , effectiveMaxResource=" + - getEffectiveMaxCapacity(CommonNodeLabelsManager.NO_LABEL); - } finally { - readLock.unlock(); - } - } - - protected String getExtendedCapacityOrWeightString() { - if (queueCapacities.getWeight() != -1) { - return "weight = " + queueCapacities.getWeight() - + " [= (float) configuredCapacity (with w suffix)] " + "\n" - + "normalizedWeight = " + queueCapacities.getNormalizedWeight() - + " [= (float) configuredCapacity / sum(configuredCapacity of " + - "all queues under the parent)]"; - } else { - return "capacity = " + queueCapacities.getCapacity() - + " [= (float) configuredCapacity / 100 ]"; - } - } - - @VisibleForTesting - public User getUser(String userName) { - return usersManager.getUser(userName); - } - - @VisibleForTesting - public User getOrCreateUser(String userName) { - return usersManager.getUserAndAddIfAbsent(userName); - } - - @Private - public List getPriorityACLs() { - readLock.lock(); - try { - return new ArrayList<>(priorityAcls); - } finally { - readLock.unlock(); - } - } - - protected void reinitialize( - CSQueue newlyParsedQueue, Resource clusterResource, - CapacitySchedulerConfiguration configuration) throws - IOException { - - writeLock.lock(); - try { - // We skip reinitialize for dynamic queues, when this is called, and - // new queue is different from this queue, we will make this queue to be - // static queue. - if (newlyParsedQueue != this) { - this.setDynamicQueue(false); - } - - // Sanity check - if (!(newlyParsedQueue instanceof LeafQueue) || !newlyParsedQueue - .getQueuePath().equals(getQueuePath())) { - throw new IOException( - "Trying to reinitialize " + getQueuePath() + " from " - + newlyParsedQueue.getQueuePath()); - } - - LeafQueue newlyParsedLeafQueue = (LeafQueue) newlyParsedQueue; - - // don't allow the maximum allocation to be decreased in size - // since we have already told running AM's the size - Resource oldMax = getMaximumAllocation(); - Resource newMax = newlyParsedLeafQueue.getMaximumAllocation(); - - if (!Resources.fitsIn(oldMax, newMax)) { - throw new IOException("Trying to reinitialize " + getQueuePath() - + " the maximum allocation size can not be decreased!" - + " Current setting: " + oldMax + ", trying to set it to: " - + newMax); - } - - setupQueueConfigs(clusterResource, configuration); - } finally { - writeLock.unlock(); - } - } - - @Override - public void reinitialize( - CSQueue newlyParsedQueue, Resource clusterResource) - throws IOException { - reinitialize(newlyParsedQueue, clusterResource, - csContext.getConfiguration()); - } - - @Override - public void submitApplicationAttempt(FiCaSchedulerApp application, - String userName) { - submitApplicationAttempt(application, userName, false); - } - - @Override - public void submitApplicationAttempt(FiCaSchedulerApp application, - String userName, boolean isMoveApp) { - // Careful! Locking order is important! - writeLock.lock(); - try { - // TODO, should use getUser, use this method just to avoid UT failure - // which is caused by wrong invoking order, will fix UT separately - User user = usersManager.getUserAndAddIfAbsent(userName); - - // Add the attempt to our data-structures - addApplicationAttempt(application, user); - } finally { - writeLock.unlock(); - } - - // We don't want to update metrics for move app - if (!isMoveApp) { - boolean unmanagedAM = application.getAppSchedulingInfo() != null && - application.getAppSchedulingInfo().isUnmanagedAM(); - usageTracker.getMetrics().submitAppAttempt(userName, unmanagedAM); - } - - parent.submitApplicationAttempt(application, userName); - } - - @Override - public void submitApplication(ApplicationId applicationId, String userName, - String queue) throws AccessControlException { - // Careful! Locking order is important! - validateSubmitApplication(applicationId, userName, queue); - - // Signal for expired auto deletion. - updateLastSubmittedTimeStamp(); - - // Inform the parent queue - try { - parent.submitApplication(applicationId, userName, queue); - } catch (AccessControlException ace) { - LOG.info("Failed to submit application to parent-queue: " + - parent.getQueuePath(), ace); - throw ace; - } - - } - - public void validateSubmitApplication(ApplicationId applicationId, - String userName, String queue) throws AccessControlException { - writeLock.lock(); - try { - // Check if the queue is accepting jobs - if (getState() != QueueState.RUNNING) { - String msg = "Queue " + getQueuePath() - + " is STOPPED. Cannot accept submission of application: " - + applicationId; - LOG.info(msg); - throw new AccessControlException(msg); - } - - // Check submission limits for queues - //TODO recalculate max applications because they can depend on capacity - if (getNumApplications() >= getMaxApplications() && !(this instanceof AutoCreatedLeafQueue)) { - String msg = - "Queue " + getQueuePath() + " already has " + getNumApplications() - + " applications," - + " cannot accept submission of application: " + applicationId; - LOG.info(msg); - throw new AccessControlException(msg); - } - - // Check submission limits for the user on this queue - User user = usersManager.getUserAndAddIfAbsent(userName); - //TODO recalculate max applications because they can depend on capacity - if (user.getTotalApplications() >= getMaxApplicationsPerUser() && !(this instanceof AutoCreatedLeafQueue)) { - String msg = "Queue " + getQueuePath() + " already has " + user - .getTotalApplications() + " applications from user " + userName - + " cannot accept submission of application: " + applicationId; - LOG.info(msg); - throw new AccessControlException(msg); - } - } finally { - writeLock.unlock(); - } - - try { - parent.validateSubmitApplication(applicationId, userName, queue); - } catch (AccessControlException ace) { - LOG.info("Failed to submit application to parent-queue: " + - parent.getQueuePath(), ace); - throw ace; - } - } - - public Resource getAMResourceLimit() { - return usageTracker.getQueueUsage().getAMLimit(); - } - - public Resource getAMResourceLimitPerPartition(String nodePartition) { - return usageTracker.getQueueUsage().getAMLimit(nodePartition); - } - - @VisibleForTesting - public Resource calculateAndGetAMResourceLimit() { - return calculateAndGetAMResourceLimitPerPartition( - RMNodeLabelsManager.NO_LABEL); - } - - @VisibleForTesting - public Resource getUserAMResourceLimit() { - return getUserAMResourceLimitPerPartition(RMNodeLabelsManager.NO_LABEL, - null); - } - - public Resource getUserAMResourceLimitPerPartition( - String nodePartition, String userName) { - float userWeight = 1.0f; - if (userName != null && getUser(userName) != null) { - userWeight = getUser(userName).getWeight(); - } - - readLock.lock(); - try { - /* - * The user am resource limit is based on the same approach as the user - * limit (as it should represent a subset of that). This means that it uses - * the absolute queue capacity (per partition) instead of the max and is - * modified by the userlimit and the userlimit factor as is the userlimit - */ - float effectiveUserLimit = Math.max(usersManager.getUserLimit() / 100.0f, - 1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1)); - float preWeightedUserLimit = effectiveUserLimit; - effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f); - - Resource queuePartitionResource = getEffectiveCapacity(nodePartition); - - Resource minimumAllocation = queueAllocationSettings.getMinimumAllocation(); - - Resource userAMLimit = Resources.multiplyAndNormalizeUp( - resourceCalculator, queuePartitionResource, - queueCapacities.getMaxAMResourcePercentage(nodePartition) - * effectiveUserLimit * usersManager.getUserLimitFactor(), - minimumAllocation); - - if (getUserLimitFactor() == -1) { - userAMLimit = Resources.multiplyAndNormalizeUp( - resourceCalculator, queuePartitionResource, - queueCapacities.getMaxAMResourcePercentage(nodePartition), - minimumAllocation); - } - - userAMLimit = - Resources.min(resourceCalculator, lastClusterResource, - userAMLimit, - Resources.clone(getAMResourceLimitPerPartition(nodePartition))); - - Resource preWeighteduserAMLimit = - Resources.multiplyAndNormalizeUp( - resourceCalculator, queuePartitionResource, - queueCapacities.getMaxAMResourcePercentage(nodePartition) - * preWeightedUserLimit * usersManager.getUserLimitFactor(), - minimumAllocation); - - if (getUserLimitFactor() == -1) { - preWeighteduserAMLimit = Resources.multiplyAndNormalizeUp( - resourceCalculator, queuePartitionResource, - queueCapacities.getMaxAMResourcePercentage(nodePartition), - minimumAllocation); - } - - preWeighteduserAMLimit = - Resources.min(resourceCalculator, lastClusterResource, - preWeighteduserAMLimit, - Resources.clone(getAMResourceLimitPerPartition(nodePartition))); - usageTracker.getQueueUsage().setUserAMLimit(nodePartition, preWeighteduserAMLimit); - - LOG.debug("Effective user AM limit for \"{}\":{}. Effective weighted" - + " user AM limit: {}. User weight: {}", userName, - preWeighteduserAMLimit, userAMLimit, userWeight); - return userAMLimit; - } finally { - readLock.unlock(); - } - - } - - public Resource calculateAndGetAMResourceLimitPerPartition( - String nodePartition) { - writeLock.lock(); - try { - /* - * For non-labeled partition, get the max value from resources currently - * available to the queue and the absolute resources guaranteed for the - * partition in the queue. For labeled partition, consider only the absolute - * resources guaranteed. Multiply this value (based on labeled/ - * non-labeled), * with per-partition am-resource-percent to get the max am - * resource limit for this queue and partition. - */ - Resource queuePartitionResource = getEffectiveCapacity(nodePartition); - - Resource queueCurrentLimit = Resources.none(); - // For non-labeled partition, we need to consider the current queue - // usage limit. - if (nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) { - synchronized (queueResourceLimitsInfo){ - queueCurrentLimit = queueResourceLimitsInfo.getQueueCurrentLimit(); - } - } - - float amResourcePercent = queueCapacities.getMaxAMResourcePercentage( - nodePartition); - - // Current usable resource for this queue and partition is the max of - // queueCurrentLimit and queuePartitionResource. - // If any of the resources available to this queue are less than queue's - // guarantee, use the guarantee as the queuePartitionUsableResource - // because nothing less than the queue's guarantee should be used when - // calculating the AM limit. - Resource queuePartitionUsableResource = (Resources.fitsIn( - resourceCalculator, queuePartitionResource, queueCurrentLimit)) ? - queueCurrentLimit : queuePartitionResource; - - Resource amResouceLimit = Resources.multiplyAndNormalizeUp( - resourceCalculator, queuePartitionUsableResource, amResourcePercent, - queueAllocationSettings.getMinimumAllocation()); - - usageTracker.getMetrics().setAMResouceLimit(nodePartition, amResouceLimit); - usageTracker.getQueueUsage().setAMLimit(nodePartition, amResouceLimit); - LOG.debug("Queue: {}, node label : {}, queue partition resource : {}," - + " queue current limit : {}, queue partition usable resource : {}," - + " amResourceLimit : {}", getQueuePath(), nodePartition, - queuePartitionResource, queueCurrentLimit, - queuePartitionUsableResource, amResouceLimit); - return amResouceLimit; - } finally { - writeLock.unlock(); - } - } - - protected void activateApplications() { - writeLock.lock(); - try { - // limit of allowed resource usage for application masters - Map userAmPartitionLimit = - new HashMap(); - - // AM Resource Limit for accessible labels can be pre-calculated. - // This will help in updating AMResourceLimit for all labels when queue - // is initialized for the first time (when no applications are present). - for (String nodePartition : getNodeLabelsForQueue()) { - calculateAndGetAMResourceLimitPerPartition(nodePartition); - } - - for (Iterator fsApp = - getPendingAppsOrderingPolicy() - .getAssignmentIterator(IteratorSelector.EMPTY_ITERATOR_SELECTOR); - fsApp.hasNext(); ) { - FiCaSchedulerApp application = fsApp.next(); - ApplicationId applicationId = application.getApplicationId(); - - // Get the am-node-partition associated with each application - // and calculate max-am resource limit for this partition. - String partitionName = application.getAppAMNodePartitionName(); - - Resource amLimit = getAMResourceLimitPerPartition(partitionName); - // Verify whether we already calculated am-limit for this label. - if (amLimit == null) { - amLimit = calculateAndGetAMResourceLimitPerPartition(partitionName); - } - // Check am resource limit. - Resource amIfStarted = Resources.add( - application.getAMResource(partitionName), - usageTracker.getQueueUsage().getAMUsed(partitionName)); - - if (LOG.isDebugEnabled()) { - LOG.debug("application " + application.getId() + " AMResource " - + application.getAMResource(partitionName) - + " maxAMResourcePerQueuePercent " + maxAMResourcePerQueuePercent - + " amLimit " + amLimit + " lastClusterResource " - + lastClusterResource + " amIfStarted " + amIfStarted - + " AM node-partition name " + partitionName); - } - - if (!resourceCalculator.fitsIn(amIfStarted, amLimit)) { - if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual( - resourceCalculator, lastClusterResource, - usageTracker.getQueueUsage().getAMUsed(partitionName), Resources.none()))) { - LOG.warn("maximum-am-resource-percent is insufficient to start a" - + " single application in queue, it is likely set too low." - + " skipping enforcement to allow at least one application" - + " to start"); - } else{ - application.updateAMContainerDiagnostics(AMState.INACTIVATED, - CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED); - LOG.debug("Not activating application {} as amIfStarted: {}" - + " exceeds amLimit: {}", applicationId, amIfStarted, amLimit); - continue; - } - } - - // Check user am resource limit - User user = usersManager.getUserAndAddIfAbsent(application.getUser()); - Resource userAMLimit = userAmPartitionLimit.get(partitionName); - - // Verify whether we already calculated user-am-limit for this label. - if (userAMLimit == null) { - userAMLimit = getUserAMResourceLimitPerPartition(partitionName, - application.getUser()); - userAmPartitionLimit.put(partitionName, userAMLimit); - } - - Resource userAmIfStarted = Resources.add( - application.getAMResource(partitionName), - user.getConsumedAMResources(partitionName)); - - if (!resourceCalculator.fitsIn(userAmIfStarted, userAMLimit)) { - if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual( - resourceCalculator, lastClusterResource, - usageTracker.getQueueUsage().getAMUsed(partitionName), Resources.none()))) { - LOG.warn("maximum-am-resource-percent is insufficient to start a" - + " single application in queue for user, it is likely set too" - + " low. skipping enforcement to allow at least one application" - + " to start"); - } else{ - application.updateAMContainerDiagnostics(AMState.INACTIVATED, - CSAMContainerLaunchDiagnosticsConstants.USER_AM_RESOURCE_LIMIT_EXCEED); - LOG.debug("Not activating application {} for user: {} as" - + " userAmIfStarted: {} exceeds userAmLimit: {}", - applicationId, user, userAmIfStarted, userAMLimit); - continue; - } - } - user.activateApplication(); - orderingPolicy.addSchedulableEntity(application); - application.updateAMContainerDiagnostics(AMState.ACTIVATED, null); - - usageTracker.getQueueUsage().incAMUsed(partitionName, - application.getAMResource(partitionName)); - user.getResourceUsage().incAMUsed(partitionName, - application.getAMResource(partitionName)); - user.getResourceUsage().setAMLimit(partitionName, userAMLimit); - usageTracker.getMetrics().incAMUsed(partitionName, application.getUser(), - application.getAMResource(partitionName)); - usageTracker.getMetrics().setAMResouceLimitForUser(partitionName, - application.getUser(), userAMLimit); - fsApp.remove(); - LOG.info("Application " + applicationId + " from user: " + application - .getUser() + " activated in queue: " + getQueuePath()); - } - } finally { - writeLock.unlock(); - } - } - - private void addApplicationAttempt(FiCaSchedulerApp application, - User user) { - writeLock.lock(); - try { - applicationAttemptMap.put(application.getApplicationAttemptId(), - application); - - if (application.isRunnable()) { - runnableApps.add(application); - LOG.debug("Adding runnable application: {}", - application.getApplicationAttemptId()); - } else { - nonRunnableApps.add(application); - LOG.info("Application attempt {} is not runnable," - + " parallel limit reached", application.getApplicationAttemptId()); - return; - } - - // Accept - user.submitApplication(); - getPendingAppsOrderingPolicy().addSchedulableEntity(application); - - // Activate applications - if (Resources.greaterThan(resourceCalculator, lastClusterResource, - lastClusterResource, Resources.none())) { - activateApplications(); - } else { - application.updateAMContainerDiagnostics(AMState.INACTIVATED, - CSAMContainerLaunchDiagnosticsConstants.CLUSTER_RESOURCE_EMPTY); - LOG.info("Skipping activateApplications for " - + application.getApplicationAttemptId() - + " since cluster resource is " + Resources.none()); - } - - LOG.info( - "Application added -" + " appId: " + application.getApplicationId() - + " user: " + application.getUser() + "," + " leaf-queue: " - + getQueuePath() + " #user-pending-applications: " + user - .getPendingApplications() + " #user-active-applications: " + user - .getActiveApplications() + " #queue-pending-applications: " - + getNumPendingApplications() + " #queue-active-applications: " - + getNumActiveApplications() - + " #queue-nonrunnable-applications: " - + getNumNonRunnableApps()); - } finally { - writeLock.unlock(); - } - } - - @Override - public void finishApplication(ApplicationId application, String user) { - // Inform the activeUsersManager - usersManager.deactivateApplication(user, application); - - appFinished(); - - // Inform the parent queue - parent.finishApplication(application, user); - } - - @Override - public void finishApplicationAttempt(FiCaSchedulerApp application, String queue) { - // Careful! Locking order is important! - removeApplicationAttempt(application, application.getUser()); - parent.finishApplicationAttempt(application, queue); - } - - private void removeApplicationAttempt( - FiCaSchedulerApp application, String userName) { - - writeLock.lock(); - try { - // TODO, should use getUser, use this method just to avoid UT failure - // which is caused by wrong invoking order, will fix UT separately - User user = usersManager.getUserAndAddIfAbsent(userName); - - boolean runnable = runnableApps.remove(application); - if (!runnable) { - // removeNonRunnableApp acquires the write lock again, which is fine - if (!removeNonRunnableApp(application)) { - LOG.error("Given app to remove " + application + - " does not exist in queue " + getQueuePath()); - } - } - - String partitionName = application.getAppAMNodePartitionName(); - boolean wasActive = orderingPolicy.removeSchedulableEntity(application); - if (!wasActive) { - pendingOrderingPolicy.removeSchedulableEntity(application); - } else{ - usageTracker.getQueueUsage().decAMUsed(partitionName, - application.getAMResource(partitionName)); - user.getResourceUsage().decAMUsed(partitionName, - application.getAMResource(partitionName)); - usageTracker.getMetrics().decAMUsed(partitionName, application.getUser(), - application.getAMResource(partitionName)); - } - applicationAttemptMap.remove(application.getApplicationAttemptId()); - - user.finishApplication(wasActive); - if (user.getTotalApplications() == 0) { - usersManager.removeUser(application.getUser()); - } - - // Check if we can activate more applications - activateApplications(); - - LOG.info( - "Application removed -" + " appId: " + application.getApplicationId() - + " user: " + application.getUser() + " queue: " + getQueuePath() - + " #user-pending-applications: " + user.getPendingApplications() - + " #user-active-applications: " + user.getActiveApplications() - + " #queue-pending-applications: " + getNumPendingApplications() - + " #queue-active-applications: " + getNumActiveApplications()); - } finally { - writeLock.unlock(); - } - } - - private FiCaSchedulerApp getApplication( - ApplicationAttemptId applicationAttemptId) { - return applicationAttemptMap.get(applicationAttemptId); - } - - private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) { - // Set preemption-allowed: - // For leaf queue, only under-utilized queue is allowed to preempt resources from other queues - if (!usageTracker.getQueueResourceQuotas().getEffectiveMinResource(nodePartition) - .equals(Resources.none())) { - limits.setIsAllowPreemption(Resources.lessThan(resourceCalculator, - csContext.getClusterResource(), usageTracker.getQueueUsage().getUsed(nodePartition), - usageTracker.getQueueResourceQuotas().getEffectiveMinResource(nodePartition))); - return; - } - - float usedCapacity = queueCapacities.getAbsoluteUsedCapacity(nodePartition); - float guaranteedCapacity = queueCapacities.getAbsoluteCapacity(nodePartition); - limits.setIsAllowPreemption(usedCapacity < guaranteedCapacity); - } - - private CSAssignment allocateFromReservedContainer(Resource clusterResource, - CandidateNodeSet candidates, - ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) { - - // Irrespective of Single / Multi Node Placement, the allocate from - // Reserved Container has to happen only for the single node which - // CapacityScheduler#allocateFromReservedContainer invokes with. - // Else In Multi Node Placement, there won't be any Allocation or - // Reserve of new containers when there is a RESERVED container on - // a node which is full. - FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates); - if (node != null) { - RMContainer reservedContainer = node.getReservedContainer(); - if (reservedContainer != null) { - FiCaSchedulerApp application = getApplication( - reservedContainer.getApplicationAttemptId()); - - if (null != application) { - ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager, - node, SystemClock.getInstance().getTime(), application); - CSAssignment assignment = application.assignContainers( - clusterResource, candidates, currentResourceLimits, - schedulingMode, reservedContainer); - return assignment; - } - } - } - - return null; - } - - private ConcurrentMap getUserLimitCache( - String partition, - SchedulingMode schedulingMode) { - synchronized (userLimitsCache) { - long latestVersion = usersManager.getLatestVersionOfUsersState(); - - if (latestVersion != this.currentUserLimitCacheVersion) { - // User limits cache needs invalidating - this.currentUserLimitCacheVersion = latestVersion; - userLimitsCache.clear(); - - Map> - uLCByPartition = new HashMap<>(); - userLimitsCache.put(partition, uLCByPartition); - - ConcurrentMap uLCBySchedulingMode = - new ConcurrentHashMap<>(); - uLCByPartition.put(schedulingMode, uLCBySchedulingMode); - - return uLCBySchedulingMode; - } - - // User limits cache does not need invalidating - Map> - uLCByPartition = userLimitsCache.get(partition); - if (uLCByPartition == null) { - uLCByPartition = new HashMap<>(); - userLimitsCache.put(partition, uLCByPartition); - } - - ConcurrentMap uLCBySchedulingMode = - uLCByPartition.get(schedulingMode); - if (uLCBySchedulingMode == null) { - uLCBySchedulingMode = new ConcurrentHashMap<>(); - uLCByPartition.put(schedulingMode, uLCBySchedulingMode); - } - - return uLCBySchedulingMode; - } - } - - @Override - public CSAssignment assignContainers(Resource clusterResource, - CandidateNodeSet candidates, - ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) { - updateCurrentResourceLimits(currentResourceLimits, clusterResource); - FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates); - - if (LOG.isDebugEnabled()) { - LOG.debug("assignContainers: partition=" + candidates.getPartition() - + " #applications=" + orderingPolicy.getNumSchedulableEntities()); - } - - setPreemptionAllowed(currentResourceLimits, candidates.getPartition()); - - // Check for reserved resources, try to allocate reserved container first. - CSAssignment assignment = allocateFromReservedContainer(clusterResource, - candidates, currentResourceLimits, schedulingMode); - if (null != assignment) { - return assignment; - } - - // if our queue cannot access this node, just return - if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY - && !queueNodeLabelsSettings.isAccessibleToPartition(candidates.getPartition())) { - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - parent.getQueuePath(), getQueuePath(), ActivityState.REJECTED, - ActivityDiagnosticConstant.QUEUE_NOT_ABLE_TO_ACCESS_PARTITION); - return CSAssignment.NULL_ASSIGNMENT; - } - - // Check if this queue need more resource, simply skip allocation if this - // queue doesn't need more resources. - if (!hasPendingResourceRequest(candidates.getPartition(), clusterResource, - schedulingMode)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Skip this queue=" + getQueuePath() - + ", because it doesn't need more resource, schedulingMode=" - + schedulingMode.name() + " node-partition=" + candidates - .getPartition()); - } - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - parent.getQueuePath(), getQueuePath(), ActivityState.SKIPPED, - ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE); - return CSAssignment.NULL_ASSIGNMENT; - } - - ConcurrentMap userLimits = - this.getUserLimitCache(candidates.getPartition(), schedulingMode); - boolean needAssignToQueueCheck = true; - IteratorSelector sel = new IteratorSelector(); - sel.setPartition(candidates.getPartition()); - for (Iterator assignmentIterator = - orderingPolicy.getAssignmentIterator(sel); - assignmentIterator.hasNext(); ) { - FiCaSchedulerApp application = assignmentIterator.next(); - - ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager, - node, SystemClock.getInstance().getTime(), application); - - // Check queue max-capacity limit - Resource appReserved = application.getCurrentReservation(); - if (needAssignToQueueCheck) { - if (!super.canAssignToThisQueue(clusterResource, - candidates.getPartition(), currentResourceLimits, appReserved, - schedulingMode)) { - ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue( - activitiesManager, node, application, application.getPriority(), - ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - parent.getQueuePath(), getQueuePath(), - ActivityState.REJECTED, - ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); - return CSAssignment.NULL_ASSIGNMENT; - } - // If there was no reservation and canAssignToThisQueue returned - // true, there is no reason to check further. - if (!this.reservationsContinueLooking - || appReserved.equals(Resources.none())) { - needAssignToQueueCheck = false; - } - } - - CachedUserLimit cul = userLimits.get(application.getUser()); - Resource cachedUserLimit = null; - if (cul != null) { - cachedUserLimit = cul.userLimit; - } - Resource userLimit = computeUserLimitAndSetHeadroom(application, - clusterResource, candidates.getPartition(), schedulingMode, - cachedUserLimit); - if (cul == null) { - cul = new CachedUserLimit(userLimit); - CachedUserLimit retVal = - userLimits.putIfAbsent(application.getUser(), cul); - if (retVal != null) { - // another thread updated the user limit cache before us - cul = retVal; - userLimit = cul.userLimit; - } - } - // Check user limit - boolean userAssignable = true; - if (!cul.canAssign && Resources.fitsIn(appReserved, cul.reservation)) { - userAssignable = false; - } else { - userAssignable = canAssignToUser(clusterResource, application.getUser(), - userLimit, application, candidates.getPartition(), - currentResourceLimits); - if (!userAssignable && Resources.fitsIn(cul.reservation, appReserved)) { - cul.canAssign = false; - cul.reservation = appReserved; - } - } - if (!userAssignable) { - application.updateAMContainerDiagnostics(AMState.ACTIVATED, - "User capacity has reached its maximum limit."); - ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue( - activitiesManager, node, application, application.getPriority(), - ActivityDiagnosticConstant.QUEUE_HIT_USER_MAX_CAPACITY_LIMIT); - continue; - } - - // Try to schedule - assignment = application.assignContainers(clusterResource, - candidates, currentResourceLimits, schedulingMode, null); - - if (LOG.isDebugEnabled()) { - LOG.debug("post-assignContainers for application " + application - .getApplicationId()); - application.showRequests(); - } - - // Did we schedule or reserve a container? - Resource assigned = assignment.getResource(); - - if (Resources.greaterThan(resourceCalculator, clusterResource, assigned, - Resources.none())) { - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - parent.getQueuePath(), getQueuePath(), - ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); - return assignment; - } else if (assignment.getSkippedType() - == CSAssignment.SkippedType.OTHER) { - ActivitiesLogger.APP.finishSkippedAppAllocationRecording( - activitiesManager, application.getApplicationId(), - ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY); - application.updateNodeInfoForAMDiagnostics(node); - } else if (assignment.getSkippedType() - == CSAssignment.SkippedType.QUEUE_LIMIT) { - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - parent.getQueuePath(), getQueuePath(), ActivityState.REJECTED, - () -> ActivityDiagnosticConstant.QUEUE_DO_NOT_HAVE_ENOUGH_HEADROOM - + " from " + application.getApplicationId()); - return assignment; - } else{ - // If we don't allocate anything, and it is not skipped by application, - // we will return to respect FIFO of applications - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - parent.getQueuePath(), getQueuePath(), ActivityState.SKIPPED, - ActivityDiagnosticConstant.QUEUE_SKIPPED_TO_RESPECT_FIFO); - ActivitiesLogger.APP.finishSkippedAppAllocationRecording( - activitiesManager, application.getApplicationId(), - ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY); - return CSAssignment.NULL_ASSIGNMENT; - } - } - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - parent.getQueuePath(), getQueuePath(), ActivityState.SKIPPED, - ActivityDiagnosticConstant.EMPTY); - - return CSAssignment.NULL_ASSIGNMENT; - } - - @Override - public boolean accept(Resource cluster, - ResourceCommitRequest request) { - ContainerAllocationProposal allocation = - request.getFirstAllocatedOrReservedContainer(); - SchedulerContainer schedulerContainer = - allocation.getAllocatedOrReservedContainer(); - - // Do not check limits when allocation from a reserved container - if (allocation.getAllocateFromReservedContainer() == null) { - readLock.lock(); - try { - FiCaSchedulerApp app = - schedulerContainer.getSchedulerApplicationAttempt(); - String username = app.getUser(); - String p = schedulerContainer.getNodePartition(); - - // check user-limit - Resource userLimit = computeUserLimitAndSetHeadroom(app, cluster, p, - allocation.getSchedulingMode(), null); - - // Deduct resources that we can release - User user = getUser(username); - if (user == null) { - LOG.debug("User {} has been removed!", username); - return false; - } - Resource usedResource = Resources.clone(user.getUsed(p)); - Resources.subtractFrom(usedResource, - request.getTotalReleasedResource()); - - if (Resources.greaterThan(resourceCalculator, cluster, usedResource, - userLimit)) { - LOG.debug("Used resource={} exceeded user-limit={}", - usedResource, userLimit); - return false; - } - } finally { - readLock.unlock(); - } - } - - return super.accept(cluster, request); - } - - private void internalReleaseContainer(Resource clusterResource, - SchedulerContainer schedulerContainer) { - RMContainer rmContainer = schedulerContainer.getRmContainer(); - - LeafQueue targetLeafQueue = - schedulerContainer.getSchedulerApplicationAttempt().getCSLeafQueue(); - - if (targetLeafQueue == this) { - // When trying to preempt containers from the same queue - if (rmContainer.getState() == RMContainerState.RESERVED) { - // For other reserved containers - // This is a reservation exchange, complete previous reserved container - completedContainer(clusterResource, - schedulerContainer.getSchedulerApplicationAttempt(), - schedulerContainer.getSchedulerNode(), rmContainer, SchedulerUtils - .createAbnormalContainerStatus(rmContainer.getContainerId(), - SchedulerUtils.UNRESERVED_CONTAINER), - RMContainerEventType.RELEASED, null, false); - } - } else{ - // When trying to preempt containers from different queue -- this - // is for lazy preemption feature (kill preemption candidate in scheduling - // cycle). - targetLeafQueue.completedContainer(clusterResource, - schedulerContainer.getSchedulerApplicationAttempt(), - schedulerContainer.getSchedulerNode(), - schedulerContainer.getRmContainer(), SchedulerUtils - .createPreemptedContainerStatus(rmContainer.getContainerId(), - SchedulerUtils.PREEMPTED_CONTAINER), - RMContainerEventType.KILL, null, false); - } - } - - private void releaseContainers(Resource clusterResource, - ResourceCommitRequest request) { - for (SchedulerContainer c : request - .getContainersToRelease()) { - internalReleaseContainer(clusterResource, c); - } - - // Handle container reservation looking, or lazy preemption case: - if (null != request.getContainersToAllocate() && !request - .getContainersToAllocate().isEmpty()) { - for (ContainerAllocationProposal context : request - .getContainersToAllocate()) { - if (null != context.getToRelease()) { - for (SchedulerContainer c : context - .getToRelease()) { - internalReleaseContainer(clusterResource, c); - } - } - } - } - } - - public void apply(Resource cluster, - ResourceCommitRequest request) { - // Do we need to call parent queue's apply? - boolean applyToParentQueue = false; - - releaseContainers(cluster, request); - - writeLock.lock(); - try { - if (request.anythingAllocatedOrReserved()) { - ContainerAllocationProposal - allocation = request.getFirstAllocatedOrReservedContainer(); - SchedulerContainer - schedulerContainer = allocation.getAllocatedOrReservedContainer(); - - // Do not modify queue when allocation from reserved container - if (allocation.getAllocateFromReservedContainer() == null) { - // Only invoke apply() of ParentQueue when new allocation / - // reservation happen. - applyToParentQueue = true; - // Book-keeping - // Note: Update headroom to account for current allocation too... - allocateResource(cluster, - schedulerContainer.getSchedulerApplicationAttempt(), - allocation.getAllocatedOrReservedResource(), - schedulerContainer.getNodePartition(), - schedulerContainer.getRmContainer()); - orderingPolicy.containerAllocated( - schedulerContainer.getSchedulerApplicationAttempt(), - schedulerContainer.getRmContainer()); - } - - // Update reserved resource - if (Resources.greaterThan(resourceCalculator, cluster, - request.getTotalReservedResource(), Resources.none())) { - incReservedResource(schedulerContainer.getNodePartition(), - request.getTotalReservedResource()); - } - } - } finally { - writeLock.unlock(); - } - - if (parent != null && applyToParentQueue) { - parent.apply(cluster, request); - } - } - - - protected Resource getHeadroom(User user, Resource queueCurrentLimit, - Resource clusterResource, FiCaSchedulerApp application) { - return getHeadroom(user, queueCurrentLimit, clusterResource, application, - RMNodeLabelsManager.NO_LABEL); - } - - protected Resource getHeadroom(User user, Resource queueCurrentLimit, - Resource clusterResource, FiCaSchedulerApp application, - String partition) { - return getHeadroom(user, queueCurrentLimit, clusterResource, - getResourceLimitForActiveUsers(application.getUser(), clusterResource, - partition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), - partition); - } - - private Resource getHeadroom(User user, - Resource currentPartitionResourceLimit, Resource clusterResource, - Resource userLimitResource, String partition) { - /** - * Headroom is: - * min( - * min(userLimit, queueMaxCap) - userConsumed, - * queueMaxCap - queueUsedResources - * ) - * - * ( which can be expressed as, - * min (userLimit - userConsumed, queuMaxCap - userConsumed, - * queueMaxCap - queueUsedResources) - * ) - * - * given that queueUsedResources >= userConsumed, this simplifies to - * - * >> min (userlimit - userConsumed, queueMaxCap - queueUsedResources) << - * - * sum of queue max capacities of multiple queue's will be greater than the - * actual capacity of a given partition, hence we need to ensure that the - * headroom is not greater than the available resource for a given partition - * - * headroom = min (unused resourcelimit of a label, calculated headroom ) - */ - currentPartitionResourceLimit = - partition.equals(RMNodeLabelsManager.NO_LABEL) - ? currentPartitionResourceLimit - : getQueueMaxResource(partition); - - Resource headroom = Resources.componentwiseMin( - Resources.subtractNonNegative(userLimitResource, - user.getUsed(partition)), - Resources.subtractNonNegative(currentPartitionResourceLimit, - usageTracker.getQueueUsage().getUsed(partition))); - // Normalize it before return - headroom = - Resources.roundDown(resourceCalculator, headroom, - queueAllocationSettings.getMinimumAllocation()); - - //headroom = min (unused resourcelimit of a label, calculated headroom ) - Resource clusterPartitionResource = - labelManager.getResourceByLabel(partition, clusterResource); - Resource clusterFreePartitionResource = - Resources.subtract(clusterPartitionResource, - csContext.getClusterResourceUsage().getUsed(partition)); - headroom = Resources.min(resourceCalculator, clusterPartitionResource, - clusterFreePartitionResource, headroom); - return headroom; - } - - private void setQueueResourceLimitsInfo( - Resource clusterResource) { - synchronized (queueResourceLimitsInfo) { - queueResourceLimitsInfo.setQueueCurrentLimit(cachedResourceLimitsForHeadroom - .getLimit()); - queueResourceLimitsInfo.setClusterResource(clusterResource); - } - } - - // It doesn't necessarily to hold application's lock here. - @Lock({LeafQueue.class}) - Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application, - Resource clusterResource, String nodePartition, - SchedulingMode schedulingMode, Resource userLimit) { - String user = application.getUser(); - User queueUser = getUser(user); - if (queueUser == null) { - LOG.debug("User {} has been removed!", user); - return Resources.none(); - } - - // Compute user limit respect requested labels, - // TODO, need consider headroom respect labels also - if (userLimit == null) { - userLimit = getResourceLimitForActiveUsers(application.getUser(), - clusterResource, nodePartition, schedulingMode); - } - setQueueResourceLimitsInfo(clusterResource); - - Resource headroom = - usageTracker.getMetrics().getUserMetrics(user) == null ? Resources.none() : - getHeadroom(queueUser, cachedResourceLimitsForHeadroom.getLimit(), - clusterResource, userLimit, nodePartition); - - if (LOG.isDebugEnabled()) { - LOG.debug("Headroom calculation for user " + user + ": " + " userLimit=" - + userLimit + " queueMaxAvailRes=" - + cachedResourceLimitsForHeadroom.getLimit() + " consumed=" - + queueUser.getUsed() + " partition=" - + nodePartition); - } - - CapacityHeadroomProvider headroomProvider = new CapacityHeadroomProvider( - queueUser, this, application, queueResourceLimitsInfo); - - application.setHeadroomProvider(headroomProvider); - - usageTracker.getMetrics().setAvailableResourcesToUser(nodePartition, user, headroom); - - return userLimit; - } - - @Lock(NoLock.class) - public int getNodeLocalityDelay() { - return nodeLocalityDelay; - } - - @Lock(NoLock.class) - public int getRackLocalityAdditionalDelay() { - return rackLocalityAdditionalDelay; - } - - @Lock(NoLock.class) - public boolean getRackLocalityFullReset() { - return rackLocalityFullReset; - } - - /** - * - * @param userName - * Name of user who has submitted one/more app to given queue. - * @param clusterResource - * total cluster resource - * @param nodePartition - * partition name - * @param schedulingMode - * scheduling mode - * RESPECT_PARTITION_EXCLUSIVITY/IGNORE_PARTITION_EXCLUSIVITY - * @return Computed User Limit - */ - public Resource getResourceLimitForActiveUsers(String userName, - Resource clusterResource, String nodePartition, - SchedulingMode schedulingMode) { - return usersManager.getComputedResourceLimitForActiveUsers(userName, - clusterResource, nodePartition, schedulingMode); - } - - /** - * - * @param userName - * Name of user who has submitted one/more app to given queue. - * @param clusterResource - * total cluster resource - * @param nodePartition - * partition name - * @param schedulingMode - * scheduling mode - * RESPECT_PARTITION_EXCLUSIVITY/IGNORE_PARTITION_EXCLUSIVITY - * @return Computed User Limit - */ - public Resource getResourceLimitForAllUsers(String userName, - Resource clusterResource, String nodePartition, - SchedulingMode schedulingMode) { - return usersManager.getComputedResourceLimitForAllUsers(userName, - clusterResource, nodePartition, schedulingMode); - } - - @Private - protected boolean canAssignToUser(Resource clusterResource, - String userName, Resource limit, FiCaSchedulerApp application, - String nodePartition, ResourceLimits currentResourceLimits) { - - readLock.lock(); - try { - User user = getUser(userName); - if (user == null) { - LOG.debug("User {} has been removed!", userName); - return false; - } - - currentResourceLimits.setAmountNeededUnreserve(Resources.none()); - - // Note: We aren't considering the current request since there is a fixed - // overhead of the AM, but it's a > check, not a >= check, so... - if (Resources.greaterThan(resourceCalculator, clusterResource, - user.getUsed(nodePartition), limit)) { - // if enabled, check to see if could we potentially use this node instead - // of a reserved node if the application has reserved containers - if (this.reservationsContinueLooking) { - if (Resources.lessThanOrEqual(resourceCalculator, clusterResource, - Resources.subtract(user.getUsed(), - application.getCurrentReservation()), limit)) { - - if (LOG.isDebugEnabled()) { - LOG.debug("User " + userName + " in queue " + getQueuePath() - + " will exceed limit based on reservations - " - + " consumed: " + user.getUsed() + " reserved: " + application - .getCurrentReservation() + " limit: " + limit); - } - Resource amountNeededToUnreserve = Resources.subtract( - user.getUsed(nodePartition), limit); - // we can only acquire a new container if we unreserve first to - // respect user-limit - currentResourceLimits.setAmountNeededUnreserve( - amountNeededToUnreserve); - return true; - } - } - if (LOG.isDebugEnabled()) { - LOG.debug("User " + userName + " in queue " + getQueuePath() - + " will exceed limit - " + " consumed: " + user - .getUsed(nodePartition) + " limit: " + limit); - } - return false; - } - return true; - } finally { - readLock.unlock(); - } - } - - @Override - protected void setDynamicQueueProperties( - CapacitySchedulerConfiguration configuration) { - // set to -1, to disable it - configuration.setUserLimitFactor(getQueuePath(), -1); - // Set Max AM percentage to a higher value - configuration.setMaximumApplicationMasterResourcePerQueuePercent( - getQueuePath(), 1f); - super.setDynamicQueueProperties(configuration); - } - - private void updateSchedulerHealthForCompletedContainer( - RMContainer rmContainer, ContainerStatus containerStatus) { - // Update SchedulerHealth for released / preempted container - SchedulerHealth schedulerHealth = csContext.getSchedulerHealth(); - if (null == schedulerHealth) { - // Only do update if we have schedulerHealth - return; - } - - if (containerStatus.getExitStatus() == ContainerExitStatus.PREEMPTED) { - schedulerHealth.updatePreemption(Time.now(), rmContainer.getAllocatedNode(), - rmContainer.getContainerId(), getQueuePath()); - schedulerHealth.updateSchedulerPreemptionCounts(1); - } else { - schedulerHealth.updateRelease(csContext.getLastNodeUpdateTime(), - rmContainer.getAllocatedNode(), rmContainer.getContainerId(), - getQueuePath()); - } - } - - /** - * Recalculate QueueUsage Ratio. - * - * @param clusterResource - * Total Cluster Resource - * @param nodePartition - * Partition - */ - public void recalculateQueueUsageRatio(Resource clusterResource, - String nodePartition) { - writeLock.lock(); - try { - ResourceUsage queueResourceUsage = getQueueResourceUsage(); - - if (nodePartition == null) { - for (String partition : Sets.union( - getQueueCapacities().getNodePartitionsSet(), - queueResourceUsage.getNodePartitionsSet())) { - usersManager.updateUsageRatio(partition, clusterResource); - } - } else { - usersManager.updateUsageRatio(nodePartition, clusterResource); - } - } finally { - writeLock.unlock(); - } - } - - @Override - public void completedContainer(Resource clusterResource, - FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer, - ContainerStatus containerStatus, RMContainerEventType event, CSQueue childQueue, - boolean sortQueues) { - // Update SchedulerHealth for released / preempted container - updateSchedulerHealthForCompletedContainer(rmContainer, containerStatus); - - if (application != null) { - boolean removed = false; - - // Careful! Locking order is important! - writeLock.lock(); - try { - Container container = rmContainer.getContainer(); - - // Inform the application & the node - // Note: It's safe to assume that all state changes to RMContainer - // happen under scheduler's lock... - // So, this is, in effect, a transaction across application & node - if (rmContainer.getState() == RMContainerState.RESERVED) { - removed = application.unreserve(rmContainer.getReservedSchedulerKey(), - node, rmContainer); - } else{ - removed = application.containerCompleted(rmContainer, containerStatus, - event, node.getPartition()); - - node.releaseContainer(rmContainer.getContainerId(), false); - } - - // Book-keeping - if (removed) { - - // Inform the ordering policy - orderingPolicy.containerReleased(application, rmContainer); - - releaseResource(clusterResource, application, container.getResource(), - node.getPartition(), rmContainer); - } - } finally { - writeLock.unlock(); - } - - - if (removed) { - // Inform the parent queue _outside_ of the leaf-queue lock - parent.completedContainer(clusterResource, application, node, - rmContainer, null, event, this, sortQueues); - } - } - - // Notify PreemptionManager - csContext.getPreemptionManager().removeKillableContainer( - new KillableContainer( - rmContainer, - node.getPartition(), - getQueuePath())); - - // Update preemption metrics if exit status is PREEMPTED - if (containerStatus != null - && ContainerExitStatus.PREEMPTED == containerStatus.getExitStatus()) { - updateQueuePreemptionMetrics(rmContainer); - } - } - - void allocateResource(Resource clusterResource, - SchedulerApplicationAttempt application, Resource resource, - String nodePartition, RMContainer rmContainer) { - writeLock.lock(); - try { - super.allocateResource(clusterResource, resource, nodePartition); - - // handle ignore exclusivity container - if (null != rmContainer && rmContainer.getNodeLabelExpression().equals( - RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals( - RMNodeLabelsManager.NO_LABEL)) { - TreeSet rmContainers = null; - if (null == (rmContainers = ignorePartitionExclusivityRMContainers.get( - nodePartition))) { - rmContainers = new TreeSet<>(); - ignorePartitionExclusivityRMContainers.put(nodePartition, - rmContainers); - } - rmContainers.add(rmContainer); - } - - // Update user metrics - String userName = application.getUser(); - - // Increment user's resource usage. - User user = usersManager.updateUserResourceUsage(userName, resource, - nodePartition, true); - - Resource partitionHeadroom = Resources.createResource(0, 0); - if (usageTracker.getMetrics().getUserMetrics(userName) != null) { - partitionHeadroom = getHeadroom(user, - cachedResourceLimitsForHeadroom.getLimit(), clusterResource, - getResourceLimitForActiveUsers(userName, clusterResource, - nodePartition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), - nodePartition); - } - usageTracker.getMetrics().setAvailableResourcesToUser(nodePartition, userName, - partitionHeadroom); - - if (LOG.isDebugEnabled()) { - LOG.debug(getQueuePath() + " user=" + userName + " used=" - + usageTracker.getQueueUsage().getUsed(nodePartition) + " numContainers=" - + usageTracker.getNumContainers() + " headroom = " + application.getHeadroom() - + " user-resources=" + user.getUsed()); - } - } finally { - writeLock.unlock(); - } - } - - void releaseResource(Resource clusterResource, - FiCaSchedulerApp application, Resource resource, String nodePartition, - RMContainer rmContainer) { - writeLock.lock(); - try { - super.releaseResource(clusterResource, resource, nodePartition); - - // handle ignore exclusivity container - if (null != rmContainer && rmContainer.getNodeLabelExpression().equals( - RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals( - RMNodeLabelsManager.NO_LABEL)) { - if (ignorePartitionExclusivityRMContainers.containsKey(nodePartition)) { - Set rmContainers = - ignorePartitionExclusivityRMContainers.get(nodePartition); - rmContainers.remove(rmContainer); - if (rmContainers.isEmpty()) { - ignorePartitionExclusivityRMContainers.remove(nodePartition); - } - } - } - - // Update user metrics - String userName = application.getUser(); - User user = usersManager.updateUserResourceUsage(userName, resource, - nodePartition, false); - - Resource partitionHeadroom = Resources.createResource(0, 0); - if (usageTracker.getMetrics().getUserMetrics(userName) != null) { - partitionHeadroom = getHeadroom(user, - cachedResourceLimitsForHeadroom.getLimit(), clusterResource, - getResourceLimitForActiveUsers(userName, clusterResource, - nodePartition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), - nodePartition); - } - usageTracker.getMetrics().setAvailableResourcesToUser(nodePartition, userName, - partitionHeadroom); - - if (LOG.isDebugEnabled()) { - LOG.debug( - getQueuePath() + " used=" + usageTracker.getQueueUsage().getUsed() + " numContainers=" - + usageTracker.getNumContainers() + " user=" + userName + " user-resources=" - + user.getUsed()); - } - } finally { - writeLock.unlock(); - } - } - - private void updateCurrentResourceLimits( - ResourceLimits currentResourceLimits, Resource clusterResource) { - // TODO: need consider non-empty node labels when resource limits supports - // node labels - // Even if ParentQueue will set limits respect child's max queue capacity, - // but when allocating reserved container, CapacityScheduler doesn't do - // this. So need cap limits by queue's max capacity here. - this.cachedResourceLimitsForHeadroom = - new ResourceLimits(currentResourceLimits.getLimit()); - Resource queueMaxResource = getEffectiveMaxCapacityDown( - RMNodeLabelsManager.NO_LABEL, queueAllocationSettings.getMinimumAllocation()); - this.cachedResourceLimitsForHeadroom.setLimit(Resources.min( - resourceCalculator, clusterResource, queueMaxResource, - currentResourceLimits.getLimit())); - } - - @Override - public void updateClusterResource(Resource clusterResource, - ResourceLimits currentResourceLimits) { - writeLock.lock(); - try { - lastClusterResource = clusterResource; - - updateAbsoluteCapacities(); - - super.updateEffectiveResources(clusterResource); - - // Update maximum applications for the queue and for users - updateMaximumApplications(csContext.getConfiguration()); - - updateCurrentResourceLimits(currentResourceLimits, clusterResource); - - // Update headroom info based on new cluster resource value - // absoluteMaxCapacity now, will be replaced with absoluteMaxAvailCapacity - // during allocation - setQueueResourceLimitsInfo(clusterResource); - - // Update user consumedRatios - recalculateQueueUsageRatio(clusterResource, null); - - // Update metrics - CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, - this, labelManager, null); - // Update configured capacity/max-capacity for default partition only - CSQueueUtils.updateConfiguredCapacityMetrics(resourceCalculator, - labelManager.getResourceByLabel(null, clusterResource), - RMNodeLabelsManager.NO_LABEL, this); - - // queue metrics are updated, more resource may be available - // activate the pending applications if possible - activateApplications(); - - // In case of any resource change, invalidate recalculateULCount to clear - // the computed user-limit. - usersManager.userLimitNeedsRecompute(); - - // Update application properties - for (FiCaSchedulerApp application : orderingPolicy - .getSchedulableEntities()) { - computeUserLimitAndSetHeadroom(application, clusterResource, - RMNodeLabelsManager.NO_LABEL, - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, null); - } - } finally { - writeLock.unlock(); - } - } - - @Override - public void incUsedResource(String nodeLabel, Resource resourceToInc, - SchedulerApplicationAttempt application) { - usersManager.updateUserResourceUsage(application.getUser(), resourceToInc, - nodeLabel, true); - super.incUsedResource(nodeLabel, resourceToInc, application); - } - - @Override - public void decUsedResource(String nodeLabel, Resource resourceToDec, - SchedulerApplicationAttempt application) { - usersManager.updateUserResourceUsage(application.getUser(), resourceToDec, - nodeLabel, false); - super.decUsedResource(nodeLabel, resourceToDec, application); - } - - public void incAMUsedResource(String nodeLabel, Resource resourceToInc, - SchedulerApplicationAttempt application) { - User user = getUser(application.getUser()); - if (user == null) { - return; - } - - user.getResourceUsage().incAMUsed(nodeLabel, - resourceToInc); - // ResourceUsage has its own lock, no addition lock needs here. - usageTracker.getQueueUsage().incAMUsed(nodeLabel, resourceToInc); - } - - public void decAMUsedResource(String nodeLabel, Resource resourceToDec, - SchedulerApplicationAttempt application) { - User user = getUser(application.getUser()); - if (user == null) { - return; - } - - user.getResourceUsage().decAMUsed(nodeLabel, - resourceToDec); - // ResourceUsage has its own lock, no addition lock needs here. - usageTracker.getQueueUsage().decAMUsed(nodeLabel, resourceToDec); - } - - @Override - public void recoverContainer(Resource clusterResource, - SchedulerApplicationAttempt attempt, RMContainer rmContainer) { - if (rmContainer.getState().equals(RMContainerState.COMPLETED)) { - return; - } - if (rmContainer.getExecutionType() != ExecutionType.GUARANTEED) { - return; - } - // Careful! Locking order is important! - writeLock.lock(); - try { - FiCaSchedulerNode node = csContext.getNode( - rmContainer.getContainer().getNodeId()); - allocateResource(clusterResource, attempt, - rmContainer.getContainer().getResource(), node.getPartition(), - rmContainer); - } finally { - writeLock.unlock(); - } - - parent.recoverContainer(clusterResource, attempt, rmContainer); - } - - /** - * Obtain (read-only) collection of pending applications. - */ - public Collection getPendingApplications() { - return Collections.unmodifiableCollection(pendingOrderingPolicy - .getSchedulableEntities()); - } - - /** - * Obtain (read-only) collection of active applications. - */ - public Collection getApplications() { - return Collections.unmodifiableCollection(orderingPolicy - .getSchedulableEntities()); - } - - /** - * Obtain (read-only) collection of all applications. - */ - public Collection getAllApplications() { - Collection apps = new HashSet( - pendingOrderingPolicy.getSchedulableEntities()); - apps.addAll(orderingPolicy.getSchedulableEntities()); - - return Collections.unmodifiableCollection(apps); - } - - /** - * Get total pending resource considering user limit for the leaf queue. This - * will be used for calculating pending resources in the preemption monitor. - * - * Consider the headroom for each user in the queue. - * Total pending for the queue = - * sum(for each user(min((user's headroom), sum(user's pending requests)))) - * NOTE: - - * @param clusterResources clusterResource - * @param partition node partition - * @param deductReservedFromPending When a container is reserved in CS, - * pending resource will not be deducted. - * This could lead to double accounting when - * doing preemption: - * In normal cases, we should deduct reserved - * resource from pending to avoid - * excessive preemption. - * @return Total pending resource considering user limit - */ - public Resource getTotalPendingResourcesConsideringUserLimit( - Resource clusterResources, String partition, - boolean deductReservedFromPending) { - readLock.lock(); - try { - Map userNameToHeadroom = - new HashMap<>(); - Resource totalPendingConsideringUserLimit = Resource.newInstance(0, 0); - for (FiCaSchedulerApp app : getApplications()) { - String userName = app.getUser(); - if (!userNameToHeadroom.containsKey(userName)) { - User user = getUsersManager().getUserAndAddIfAbsent(userName); - Resource headroom = Resources.subtract( - getResourceLimitForActiveUsers(app.getUser(), clusterResources, - partition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), - user.getUsed(partition)); - // Make sure headroom is not negative. - headroom = Resources.componentwiseMax(headroom, Resources.none()); - userNameToHeadroom.put(userName, headroom); - } - - // Check if we need to deduct reserved from pending - Resource pending = app.getAppAttemptResourceUsage().getPending( - partition); - if (deductReservedFromPending) { - pending = Resources.subtract(pending, - app.getAppAttemptResourceUsage().getReserved(partition)); - } - pending = Resources.componentwiseMax(pending, Resources.none()); - - Resource minpendingConsideringUserLimit = Resources.componentwiseMin( - userNameToHeadroom.get(userName), pending); - Resources.addTo(totalPendingConsideringUserLimit, - minpendingConsideringUserLimit); - Resources.subtractFrom(userNameToHeadroom.get(userName), - minpendingConsideringUserLimit); - } - return totalPendingConsideringUserLimit; - } finally { - readLock.unlock(); - } - - } - - @Override - public void collectSchedulerApplications( - Collection apps) { - readLock.lock(); - try { - for (FiCaSchedulerApp pendingApp : pendingOrderingPolicy - .getSchedulableEntities()) { - apps.add(pendingApp.getApplicationAttemptId()); - } - for (FiCaSchedulerApp app : orderingPolicy.getSchedulableEntities()) { - apps.add(app.getApplicationAttemptId()); - } - } finally { - readLock.unlock(); - } - - } - - @Override - public void attachContainer(Resource clusterResource, - FiCaSchedulerApp application, RMContainer rmContainer) { - if (application != null && rmContainer != null - && rmContainer.getExecutionType() == ExecutionType.GUARANTEED) { - FiCaSchedulerNode node = - csContext.getNode(rmContainer.getContainer().getNodeId()); - allocateResource(clusterResource, application, rmContainer.getContainer() - .getResource(), node.getPartition(), rmContainer); - LOG.info("movedContainer" + " container=" + rmContainer.getContainer() - + " containerState="+ rmContainer.getState() - + " resource=" + rmContainer.getContainer().getResource() - + " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity() - + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" - + usageTracker.getQueueUsage().getUsed() + " cluster=" + clusterResource); - // Inform the parent queue - parent.attachContainer(clusterResource, application, rmContainer); - } - } - - @Override - public void detachContainer(Resource clusterResource, - FiCaSchedulerApp application, RMContainer rmContainer) { - if (application != null && rmContainer != null - && rmContainer.getExecutionType() == ExecutionType.GUARANTEED) { - FiCaSchedulerNode node = - csContext.getNode(rmContainer.getContainer().getNodeId()); - releaseResource(clusterResource, application, rmContainer.getContainer() - .getResource(), node.getPartition(), rmContainer); - LOG.info("movedContainer" + " container=" + rmContainer.getContainer() - + " containerState="+ rmContainer.getState() - + " resource=" + rmContainer.getContainer().getResource() - + " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity() - + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" - + usageTracker.getQueueUsage().getUsed() + " cluster=" + clusterResource); - // Inform the parent queue - parent.detachContainer(clusterResource, application, rmContainer); - } - } - - /** - * @return all ignored partition exclusivity RMContainers in the LeafQueue, - * this will be used by preemption policy. - */ - public Map> - getIgnoreExclusivityRMContainers() { - Map> clonedMap = new HashMap<>(); - - readLock.lock(); - try { - for (Map.Entry> entry : ignorePartitionExclusivityRMContainers - .entrySet()) { - clonedMap.put(entry.getKey(), new TreeSet<>(entry.getValue())); - } - - return clonedMap; - - } finally { - readLock.unlock(); - } - } - - public void setCapacity(float capacity) { - queueCapacities.setCapacity(capacity); - } - - public void setCapacity(String nodeLabel, float capacity) { - queueCapacities.setCapacity(nodeLabel, capacity); - } - - public void setAbsoluteCapacity(float absoluteCapacity) { - queueCapacities.setAbsoluteCapacity(absoluteCapacity); - } - - public void setAbsoluteCapacity(String nodeLabel, float absoluteCapacity) { - queueCapacities.setAbsoluteCapacity(nodeLabel, absoluteCapacity); - } - - public void setMaxApplicationsPerUser(int maxApplicationsPerUser) { - this.maxApplicationsPerUser = maxApplicationsPerUser; - } - - public void setMaxApplications(int maxApplications) { - this.maxApplications = maxApplications; - } - - public void setMaxAMResourcePerQueuePercent( - float maxAMResourcePerQueuePercent) { - this.maxAMResourcePerQueuePercent = maxAMResourcePerQueuePercent; - } - - public OrderingPolicy - getOrderingPolicy() { - return orderingPolicy; - } - - void setOrderingPolicy( - OrderingPolicy orderingPolicy) { - writeLock.lock(); - try { - if (null != this.orderingPolicy) { - orderingPolicy.addAllSchedulableEntities( - this.orderingPolicy.getSchedulableEntities()); - } - this.orderingPolicy = orderingPolicy; - } finally { - writeLock.unlock(); - } - } - - @Override - public Priority getDefaultApplicationPriority() { - return defaultAppPriorityPerQueue; - } - - public void updateApplicationPriority(SchedulerApplication app, - Priority newAppPriority) { - writeLock.lock(); - try { - FiCaSchedulerApp attempt = app.getCurrentAppAttempt(); - boolean isActive = orderingPolicy.removeSchedulableEntity(attempt); - if (!isActive) { - pendingOrderingPolicy.removeSchedulableEntity(attempt); - } - // Update new priority in SchedulerApplication - attempt.setPriority(newAppPriority); - - if (isActive) { - orderingPolicy.addSchedulableEntity(attempt); - } else { - pendingOrderingPolicy.addSchedulableEntity(attempt); - } - } finally { - writeLock.unlock(); - } - } - - public OrderingPolicy - getPendingAppsOrderingPolicy() { - return pendingOrderingPolicy; - } - - /* - * Holds shared values used by all applications in - * the queue to calculate headroom on demand - */ - static class QueueResourceLimitsInfo { - private Resource queueCurrentLimit; - private Resource clusterResource; - - public void setQueueCurrentLimit(Resource currentLimit) { - this.queueCurrentLimit = currentLimit; - } - - public Resource getQueueCurrentLimit() { - return queueCurrentLimit; - } - - public void setClusterResource(Resource clusterResource) { - this.clusterResource = clusterResource; - } - - public Resource getClusterResource() { - return clusterResource; - } - } - - @Override - public void stopQueue() { - writeLock.lock(); - try { - if (getNumApplications() > 0) { - updateQueueState(QueueState.DRAINING); - } else { - updateQueueState(QueueState.STOPPED); - } - } finally { - writeLock.unlock(); - } - } - - void updateMaximumApplications(CapacitySchedulerConfiguration conf) { - int maxAppsForQueue = conf.getMaximumApplicationsPerQueue(getQueuePath()); - - int maxDefaultPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue(); - int maxSystemApps = conf.getMaximumSystemApplications(); - int baseMaxApplications = maxDefaultPerQueueApps > 0 ? - Math.min(maxDefaultPerQueueApps, maxSystemApps) - : maxSystemApps; - - String maxLabel = RMNodeLabelsManager.NO_LABEL; - if (maxAppsForQueue < 0) { - if (maxDefaultPerQueueApps > 0 && this.capacityConfigType - != CapacityConfigType.ABSOLUTE_RESOURCE) { - maxAppsForQueue = baseMaxApplications; - } else { - for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { - int maxApplicationsByLabel = (int) (baseMaxApplications - * queueCapacities.getAbsoluteCapacity(label)); - if (maxApplicationsByLabel > maxAppsForQueue) { - maxAppsForQueue = maxApplicationsByLabel; - maxLabel = label; - } - } - } - } - - setMaxApplications(maxAppsForQueue); - - updateMaxAppsPerUser(); - - LOG.info("LeafQueue:" + getQueuePath() + - "update max app related, maxApplications=" - + maxAppsForQueue + ", maxApplicationsPerUser=" - + maxApplicationsPerUser + ", Abs Cap:" + queueCapacities - .getAbsoluteCapacity(maxLabel) + ", Cap: " + queueCapacities - .getCapacity(maxLabel) + ", MaxCap : " + queueCapacities - .getMaximumCapacity(maxLabel)); - } - - private void updateMaxAppsPerUser() { - int maxAppsPerUser = maxApplications; - if (getUsersManager().getUserLimitFactor() != -1) { - int maxApplicationsWithUserLimits = (int) (maxApplications - * (getUsersManager().getUserLimit() / 100.0f) - * getUsersManager().getUserLimitFactor()); - maxAppsPerUser = Math.min(maxApplications, - maxApplicationsWithUserLimits); - } - - setMaxApplicationsPerUser(maxAppsPerUser); - } - - /** - * Get all valid users in this queue. - * @return user list - */ - public Set getAllUsers() { - return this.getUsersManager().getUsers().keySet(); - } - - static class CachedUserLimit { - final Resource userLimit; - volatile boolean canAssign = true; - volatile Resource reservation = Resources.none(); - - CachedUserLimit(Resource userLimit) { - this.userLimit = userLimit; - } - } - - private void updateQueuePreemptionMetrics(RMContainer rmc) { - final long usedMillis = rmc.getFinishTime() - rmc.getCreationTime(); - final long usedSeconds = usedMillis / DateUtils.MILLIS_PER_SECOND; - CSQueueMetrics metrics = usageTracker.getMetrics(); - Resource containerResource = rmc.getAllocatedResource(); - metrics.preemptContainer(); - long mbSeconds = (containerResource.getMemorySize() * usedMillis) - / DateUtils.MILLIS_PER_SECOND; - long vcSeconds = (containerResource.getVirtualCores() * usedMillis) - / DateUtils.MILLIS_PER_SECOND; - metrics.updatePreemptedMemoryMBSeconds(mbSeconds); - metrics.updatePreemptedVcoreSeconds(vcSeconds); - metrics.updatePreemptedResources(containerResource); - metrics.updatePreemptedSecondsForCustomResources(containerResource, - usedSeconds); - metrics.updatePreemptedForCustomResources(containerResource); - } - - @Override - int getNumRunnableApps() { - readLock.lock(); - try { - return runnableApps.size(); - } finally { - readLock.unlock(); - } - } - - int getNumNonRunnableApps() { - readLock.lock(); - try { - return nonRunnableApps.size(); - } finally { - readLock.unlock(); - } - } - - boolean removeNonRunnableApp(FiCaSchedulerApp app) { - writeLock.lock(); - try { - return nonRunnableApps.remove(app); - } finally { - writeLock.unlock(); - } - } - - List getCopyOfNonRunnableAppSchedulables() { - List appsToReturn = new ArrayList<>(); - readLock.lock(); - try { - appsToReturn.addAll(nonRunnableApps); - } finally { - readLock.unlock(); - } - return appsToReturn; - } - - @Override - public boolean isEligibleForAutoDeletion() { - return isDynamicQueue() && getNumApplications() == 0 - && csContext.getConfiguration(). - isAutoExpiredDeletionEnabled(this.getQueuePath()); + LOG.debug("LeafQueue: name={}, fullname={}", queueName, getQueuePath()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index 6e7325c3a9747e..ddfb24bf6fce69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -334,7 +334,7 @@ public List getScheduleableApplications() { try { List apps = new ArrayList<>(); for (CSQueue childQueue : getChildQueues()) { - apps.addAll(((LeafQueue) childQueue).getApplications()); + apps.addAll(((AbstractLeafQueue) childQueue).getApplications()); } return Collections.unmodifiableList(apps); } finally { @@ -347,7 +347,7 @@ public List getPendingApplications() { try { List apps = new ArrayList<>(); for (CSQueue childQueue : getChildQueues()) { - apps.addAll(((LeafQueue) childQueue).getPendingApplications()); + apps.addAll(((AbstractLeafQueue) childQueue).getPendingApplications()); } return Collections.unmodifiableList(apps); } finally { @@ -360,7 +360,7 @@ public List getAllApplications() { try { List apps = new ArrayList<>(); for (CSQueue childQueue : getChildQueues()) { - apps.addAll(((LeafQueue) childQueue).getAllApplications()); + apps.addAll(((AbstractLeafQueue) childQueue).getAllApplications()); } return Collections.unmodifiableList(apps); } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index aec2bd8468db55..43391897cdfad9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -675,10 +675,10 @@ public void reinitialize(CSQueue newlyParsedQueue, // parent Queue has been converted to child queue. The CS has already // checked to ensure that this child-queue is in STOPPED state if // Child queue has been converted to ParentQueue. - if ((childQueue instanceof LeafQueue + if ((childQueue instanceof AbstractLeafQueue && newChildQueue instanceof ParentQueue) || (childQueue instanceof ParentQueue - && newChildQueue instanceof LeafQueue)) { + && newChildQueue instanceof AbstractLeafQueue)) { // We would convert this LeafQueue to ParentQueue, or vice versa. // consider this as the combination of DELETE then ADD. newChildQueue.setParent(this); @@ -1134,7 +1134,7 @@ private CSAssignment assignContainersToChildQueues(Resource cluster, assignment = childAssignment; } Resource blockedHeadroom = null; - if (childQueue instanceof LeafQueue) { + if (childQueue instanceof AbstractLeafQueue) { blockedHeadroom = childLimits.getHeadroom(); } else { blockedHeadroom = childLimits.getBlockedHeadroom(); @@ -1548,7 +1548,7 @@ private void killContainersToEnforceMaxQueueCapacity(String partition, FiCaSchedulerNode node = csContext.getNode( toKillContainer.getAllocatedNode()); if (null != attempt && null != node) { - LeafQueue lq = attempt.getCSLeafQueue(); + AbstractLeafQueue lq = attempt.getCSLeafQueue(); lq.completedContainer(clusterResource, attempt, node, toKillContainer, SchedulerUtils.createPreemptedContainerStatus( toKillContainer.getContainerId(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java index 5a8ce9ac487b24..4208bf06a24a98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java @@ -40,6 +40,11 @@ public class ReservationQueue extends AbstractAutoCreatedLeafQueue { public ReservationQueue(CapacitySchedulerContext cs, String queueName, PlanQueue parent) throws IOException { super(cs, queueName, parent, null); + super.setupQueueConfigs(cs.getClusterResource(), + cs.getConfiguration()); + + LOG.debug("Initialized ReservationQueue: name={}, fullname={}", + queueName, getQueuePath()); // the following parameters are common to all reservation in the plan updateQuotas(parent.getUserLimitForReservation(), parent.getUserLimitFactor(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java index 94df9ab22c4356..73aad3c177193b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java @@ -55,7 +55,7 @@ public class UsersManager implements AbstractUsersManager { /* * Member declaration for UsersManager class. */ - private final LeafQueue lQueue; + private final AbstractLeafQueue lQueue; private final RMNodeLabelsManager labelManager; private final ResourceCalculator resourceCalculator; private final CapacitySchedulerContext scheduler; @@ -301,7 +301,7 @@ public void setWeight(float weight) { * @param resourceCalculator * rc */ - public UsersManager(QueueMetrics metrics, LeafQueue lQueue, + public UsersManager(QueueMetrics metrics, AbstractLeafQueue lQueue, RMNodeLabelsManager labelManager, CapacitySchedulerContext scheduler, ResourceCalculator resourceCalculator) { ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java index 76ab7cab7ac580..7458df904518fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java @@ -20,35 +20,23 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler - .SchedulerDynamicEditException; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .AbstractAutoCreatedLeafQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .AutoCreatedLeafQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .AutoCreatedLeafQueueConfig; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .AutoCreatedQueueManagementPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractAutoCreatedLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueueConfig; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueManagementPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .CapacitySchedulerContext; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .LeafQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .ManagedParentQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .ParentQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .QueueCapacities; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .QueueManagementChange; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica - .FiCaSchedulerApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueManagementChange; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.MonotonicClock; import org.apache.hadoop.yarn.util.resource.Resources; @@ -136,7 +124,7 @@ private boolean addLeafQueueStateIfNotExists(String leafQueuePath, return false; } - public boolean createLeafQueueStateIfNotExists(LeafQueue leafQueue, + public boolean createLeafQueueStateIfNotExists(AbstractLeafQueue leafQueue, String partition) { return addLeafQueueStateIfNotExists(leafQueue.getQueuePath(), partition, new LeafQueueStatePerPartition()); @@ -482,9 +470,9 @@ void updateLeafQueueState() { Set newQueues = new HashSet<>(); for (CSQueue newQueue : managedParentQueue.getChildQueues()) { - if (newQueue instanceof LeafQueue) { + if (newQueue instanceof AbstractLeafQueue) { for (String nodeLabel : leafQueueTemplateNodeLabels) { - leafQueueState.createLeafQueueStateIfNotExists((LeafQueue) newQueue, + leafQueueState.createLeafQueueStateIfNotExists((AbstractLeafQueue) newQueue, nodeLabel); newPartitions.add(nodeLabel); } @@ -590,7 +578,7 @@ private Map deactivateLeafQueuesIfInActive( if (leafQueue != null) { if (isActive(leafQueue, nodeLabel) && !hasPendingApps(leafQueue)) { QueueCapacities capacities = leafQueueEntitlements.getCapacityOfQueue(leafQueue); - updateToZeroCapacity(capacities, nodeLabel, (LeafQueue)childQueue); + updateToZeroCapacity(capacities, nodeLabel, (AbstractLeafQueue) childQueue); deactivatedQueues.put(leafQueue.getQueuePath(), leafQueueTemplateCapacities); } } else { @@ -780,7 +768,7 @@ public AutoCreatedLeafQueueConfig getInitialLeafQueueConfiguration( } private void updateToZeroCapacity(QueueCapacities capacities, - String nodeLabel, LeafQueue leafQueue) { + String nodeLabel, AbstractLeafQueue leafQueue) { capacities.setCapacity(nodeLabel, 0.0f); capacities.setMaximumCapacity(nodeLabel, leafQueueTemplateCapacities.getMaximumCapacity(nodeLabel)); @@ -801,7 +789,7 @@ private void updateCapacityFromTemplate(QueueCapacities capacities, } @VisibleForTesting - LeafQueueStatePerPartition getLeafQueueState(LeafQueue queue, + LeafQueueStatePerPartition getLeafQueueState(AbstractLeafQueue queue, String partition) throws SchedulerDynamicEditException { readLock.lock(); try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 011a2546ebc10b..3a0fd347e5a0b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -66,11 +66,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAMContainerLaunchDiagnosticsConstants; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityHeadroomProvider; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.AbstractContainerAllocator; @@ -958,8 +958,8 @@ public RMContainer findNodeToUnreserve(FiCaSchedulerNode node, } } - public LeafQueue getCSLeafQueue() { - return (LeafQueue)queue; + public AbstractLeafQueue getCSLeafQueue() { + return (AbstractLeafQueue)queue; } public CSAssignment assignContainers(Resource clusterResource, @@ -996,7 +996,7 @@ public void nodePartitionUpdated(RMContainer rmContainer, String oldPartition, protected void getPendingAppDiagnosticMessage( StringBuilder diagnosticMessage) { - LeafQueue queue = getCSLeafQueue(); + AbstractLeafQueue queue = getCSLeafQueue(); diagnosticMessage.append(" Details : AM Partition = ") .append(appAMNodePartitionName.isEmpty() ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName) @@ -1019,7 +1019,7 @@ appAMNodePartitionName, getUser())) protected void getActivedAppDiagnosticMessage( StringBuilder diagnosticMessage) { - LeafQueue queue = getCSLeafQueue(); + AbstractLeafQueue queue = getCSLeafQueue(); QueueCapacities queueCapacities = queue.getQueueCapacities(); QueueResourceQuotas queueResourceQuotas = queue.getQueueResourceQuotas(); diagnosticMessage.append(" Details : AM Partition = ") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 74c7c2073b0e47..3d410ecddafad4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -25,10 +25,10 @@ import javax.xml.bind.annotation.XmlType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper; @@ -179,7 +179,7 @@ protected CapacitySchedulerQueueInfoList getQueues( List childLeafQueues = new ArrayList<>(); List childNonLeafQueues = new ArrayList<>(); for (CSQueue queue : parent.getChildQueues()) { - if (queue instanceof LeafQueue) { + if (queue instanceof AbstractLeafQueue) { childLeafQueues.add(queue); } else { childNonLeafQueues.add(queue); @@ -190,8 +190,8 @@ protected CapacitySchedulerQueueInfoList getQueues( for (CSQueue queue : childQueues) { CapacitySchedulerQueueInfo info; - if (queue instanceof LeafQueue) { - info = new CapacitySchedulerLeafQueueInfo(cs, (LeafQueue) queue); + if (queue instanceof AbstractLeafQueue) { + info = new CapacitySchedulerLeafQueueInfo(cs, (AbstractLeafQueue) queue); } else { info = new CapacitySchedulerQueueInfo(cs, queue); info.queues = getQueues(cs, queue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index 4e9ced8beeaaf9..5b1da1977bf55c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -27,10 +27,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .AutoCreatedLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; @@ -63,7 +63,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { CapacitySchedulerLeafQueueInfo() { }; - CapacitySchedulerLeafQueueInfo(CapacityScheduler cs, LeafQueue q) { + CapacitySchedulerLeafQueueInfo(CapacityScheduler cs, AbstractLeafQueue q) { super(cs, q); numActiveApplications = q.getNumActiveApplications(); numPendingApplications = q.getNumPendingApplications(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java index 8b3602da31e1dd..0ba9bbb8418175 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java @@ -18,9 +18,9 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AutoQueueTemplatePropertiesInfo; @@ -82,7 +82,7 @@ public static String getMode(CSQueue queue) throws YarnRuntimeException { } public static String getQueueType(CSQueue queue) { - if (queue instanceof LeafQueue) { + if (queue instanceof AbstractLeafQueue) { return LEAF_QUEUE; } else if (queue instanceof ParentQueue) { return PARENT_QUEUE; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index c5f45fd9789a5e..52a34fbf761617 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; @@ -445,7 +446,7 @@ private void checkCSQueue(MockRM rm, checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource, numContainers); - LeafQueue queue = (LeafQueue) app.getQueue(); + AbstractLeafQueue queue = (AbstractLeafQueue) app.getQueue(); Resource availableResources = Resources.subtract(queueResource, usedResource); // ************ check app headroom **************** @@ -470,7 +471,7 @@ private void checkCSLeafQueue(MockRM rm, SchedulerApplication app, Resource clusterResource, Resource queueResource, Resource usedResource, int numContainers) { - LeafQueue leafQueue = (LeafQueue) app.getQueue(); + AbstractLeafQueue leafQueue = (AbstractLeafQueue) app.getQueue(); // assert queue used resources. assertEquals(usedResource, leafQueue.getUsedResources()); assertEquals(numContainers, leafQueue.getNumContainers()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index 23332836f80f55..82c534e6aad79e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -587,7 +587,6 @@ public void testPreemptedAMRestartOnRMRestart() throws Exception { getConf().set( YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); getConf().setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); - getConf().setInt(YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL, 1); MockRM rm1 = new MockRM(getConf()); MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java index 33b9eecb15f9da..63f007b45b923e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java @@ -203,8 +203,6 @@ private static YarnConfiguration getConf(boolean v1Enabled, MemoryTimelineStore.class, TimelineStore.class); yarnConf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS, MemoryTimelineStateStore.class, TimelineStateStore.class); - yarnConf.setInt(YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL, - 1); } if (v2Enabled) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index a9a57314cc4208..7bea24c8416c95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -18,10 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -30,6 +27,14 @@ import java.util.Map; import java.util.Set; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -65,19 +70,31 @@ import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore; import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(Parameterized.class) public class TestSystemMetricsPublisher { + @Parameters + public static Collection data() { + return Arrays.asList(new Object[][] {{false, 0}, {true, 1}}); + } + private static ApplicationHistoryServer timelineServer; private static TimelineServiceV1Publisher metricsPublisher; private static TimelineStore store; - @BeforeClass - public static void setup() throws Exception { + @Parameterized.Parameter + public boolean rmTimelineServerV1PublisherBatchEnabled; + + @Parameterized.Parameter(1) + public int rmTimelineServerV1PublisherInterval; + + @Before + public void setup() throws Exception { YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true); @@ -88,8 +105,10 @@ public static void setup() throws Exception { conf.setInt( YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE, 2); + conf.setBoolean(YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED, + rmTimelineServerV1PublisherBatchEnabled); conf.setInt(YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL, - 1); + rmTimelineServerV1PublisherInterval); timelineServer = new ApplicationHistoryServer(); timelineServer.init(conf); @@ -101,8 +120,8 @@ public static void setup() throws Exception { metricsPublisher.start(); } - @AfterClass - public static void tearDown() throws Exception { + @After + public void tearDown() throws Exception { if (metricsPublisher != null) { metricsPublisher.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java index 43347c76cc95c6..b560d9798e2a5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java @@ -139,7 +139,7 @@ public long getStartTime() { } private void removeApp(FiCaSchedulerApp attempt) { - LeafQueue queue = attempt.getCSLeafQueue(); + AbstractLeafQueue queue = attempt.getCSLeafQueue(); queue.finishApplicationAttempt(attempt, queue.getQueuePath()); maxAppsEnforcer.untrackApp(attempt); maxAppsEnforcer.updateRunnabilityOnAppRemoval(attempt); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index 3d40ccf26c6a65..b5eaf3ca76693e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -685,7 +685,7 @@ public void testAutoCreatedQueueTemplateConfig() throws Exception { "root.a.*") + "capacity", "6w"); cs.reinitialize(csConf, mockRM.getRMContext()); - LeafQueue a2 = createQueue("root.a.a-auto.a2"); + AbstractLeafQueue a2 = createQueue("root.a.a-auto.a2"); Assert.assertEquals("weight is not set by template", 6f, a2.getQueueCapacities().getWeight(), 1e-6); Assert.assertEquals("user limit factor should be disabled with dynamic queues", @@ -719,7 +719,7 @@ public void testAutoCreatedQueueTemplateConfig() throws Exception { "root.a") + CapacitySchedulerConfiguration .AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE, false); cs.reinitialize(csConf, mockRM.getRMContext()); - LeafQueue a3 = createQueue("root.a.a3"); + AbstractLeafQueue a3 = createQueue("root.a.a3"); Assert.assertFalse("auto queue deletion should be turned off on a3", a3.isEligibleForAutoDeletion()); @@ -729,27 +729,26 @@ public void testAutoCreatedQueueTemplateConfig() throws Exception { csConf.setQueues("root", new String[]{"a", "b", "c"}); csConf.setAutoQueueCreationV2Enabled("root.c", true); cs.reinitialize(csConf, mockRM.getRMContext()); - LeafQueue c1 = createQueue("root.c.c1"); + AbstractLeafQueue c1 = createQueue("root.c.c1"); Assert.assertEquals("weight is not set for label TEST", 6f, c1.getQueueCapacities().getWeight("TEST"), 1e-6); cs.reinitialize(csConf, mockRM.getRMContext()); - c1 = (LeafQueue) cs.getQueue("root.c.c1"); + c1 = (AbstractLeafQueue) cs.getQueue("root.c.c1"); Assert.assertEquals("weight is not set for label TEST", 6f, c1.getQueueCapacities().getWeight("TEST"), 1e-6); - } @Test public void testAutoCreatedQueueConfigChange() throws Exception { startScheduler(); - LeafQueue a2 = createQueue("root.a.a-auto.a2"); + AbstractLeafQueue a2 = createQueue("root.a.a-auto.a2"); csConf.setNonLabeledQueueWeight("root.a.a-auto.a2", 4f); cs.reinitialize(csConf, mockRM.getRMContext()); Assert.assertEquals("weight is not explicitly set", 4f, a2.getQueueCapacities().getWeight(), 1e-6); - a2 = (LeafQueue) cs.getQueue("root.a.a-auto.a2"); + a2 = (AbstractLeafQueue) cs.getQueue("root.a.a-auto.a2"); csConf.setState("root.a.a-auto.a2", QueueState.STOPPED); cs.reinitialize(csConf, mockRM.getRMContext()); Assert.assertEquals("root.a.a-auto.a2 has not been stopped", @@ -1223,7 +1222,7 @@ public void testParentQueueDynamicChildRemoval() throws Exception { Assert.assertNull("root.e.e1-auto should have been removed", eAuto); } - protected LeafQueue createQueue(String queuePath) throws YarnException, + protected AbstractLeafQueue createQueue(String queuePath) throws YarnException, IOException { return autoQueueHandler.createQueue(new QueuePath(queuePath)); } From 290a68586c1d9249da80a1af08bcc1bb65de4625 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Mon, 13 Dec 2021 21:57:46 +0100 Subject: [PATCH 10/33] YARN-10907. Minimize usages of AbstractCSQueue#csContext. Contributed by Benjamin Teke --- .../CapacitySchedulerPlanFollower.java | 4 +- .../AbstractAutoCreatedLeafQueue.java | 24 +- .../scheduler/capacity/AbstractCSQueue.java | 100 +-- .../scheduler/capacity/AbstractLeafQueue.java | 712 ++++++++++-------- .../capacity/AbstractManagedParentQueue.java | 13 +- .../capacity/AutoCreatedLeafQueue.java | 24 +- .../AutoCreatedQueueManagementPolicy.java | 10 +- .../capacity/CSQueuePreemptionSettings.java | 24 +- .../scheduler/capacity/CapacityScheduler.java | 9 + .../CapacitySchedulerConfiguration.java | 6 + .../capacity/CapacitySchedulerContext.java | 2 + .../CapacitySchedulerQueueContext.java | 132 ++++ .../CapacitySchedulerQueueManager.java | 31 +- .../scheduler/capacity/LeafQueue.java | 20 +- .../capacity/ManagedParentQueue.java | 88 +-- .../scheduler/capacity/ParentQueue.java | 101 ++- .../scheduler/capacity/PlanQueue.java | 10 +- .../capacity/QueueAllocationSettings.java | 17 +- .../capacity/QueueNodeLabelsSettings.java | 25 +- .../scheduler/capacity/ReservationQueue.java | 15 +- .../scheduler/capacity/UsersManager.java | 18 +- .../allocator/RegularContainerAllocator.java | 6 +- ...uaranteedOrZeroCapacityOverTimePolicy.java | 18 +- .../resourcemanager/TestAppManager.java | 2 +- .../TestAbsoluteResourceConfiguration.java | 4 +- .../capacity/TestApplicationLimits.java | 121 ++- .../TestApplicationLimitsByPartition.java | 15 +- .../TestCSMaxRunningAppsEnforcer.java | 4 + .../scheduler/capacity/TestCSQueueStore.java | 11 +- ...estCapacitySchedulerAutoQueueCreation.java | 6 +- .../TestCapacitySchedulerDynamicBehavior.java | 14 +- .../capacity/TestCapacitySchedulerPerf.java | 2 +- .../capacity/TestChildQueueOrder.java | 8 +- .../scheduler/capacity/TestLeafQueue.java | 54 +- .../scheduler/capacity/TestParentQueue.java | 57 +- .../capacity/TestReservationQueue.java | 9 +- .../scheduler/capacity/TestReservations.java | 24 +- .../scheduler/capacity/TestUsersManager.java | 4 - 38 files changed, 1009 insertions(+), 735 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java index 7962d8e30f55fd..910079171fa2f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java @@ -95,7 +95,7 @@ protected void addReservationQueue( PlanQueue planQueue = (PlanQueue)queue; try { ReservationQueue resQueue = - new ReservationQueue(cs, currResId, planQueue); + new ReservationQueue(cs.getQueueContext(), currResId, planQueue); cs.addQueue(resQueue); } catch (SchedulerDynamicEditException e) { LOG.warn( @@ -115,7 +115,7 @@ protected void createDefaultReservationQueue( if (cs.getQueue(defReservationId) == null) { try { ReservationQueue defQueue = - new ReservationQueue(cs, defReservationId, planQueue); + new ReservationQueue(cs.getQueueContext(), defReservationId, planQueue); cs.addQueue(defQueue); } catch (SchedulerDynamicEditException e) { LOG.warn( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java index 36d2aef4806ed8..15960c8de8494b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java @@ -35,23 +35,15 @@ * of AbstractManagedParentQueue */ public class AbstractAutoCreatedLeafQueue extends AbstractLeafQueue { + private static final Logger LOG = LoggerFactory.getLogger( + AbstractAutoCreatedLeafQueue.class); protected AbstractManagedParentQueue parent; - public AbstractAutoCreatedLeafQueue(CapacitySchedulerContext cs, + public AbstractAutoCreatedLeafQueue(CapacitySchedulerQueueContext queueContext, String queueName, AbstractManagedParentQueue parent, CSQueue old) throws IOException { - super(cs, queueName, parent, old); - this.parent = parent; - } - - private static final Logger LOG = LoggerFactory.getLogger( - AbstractAutoCreatedLeafQueue.class); - - public AbstractAutoCreatedLeafQueue(CapacitySchedulerContext cs, - CapacitySchedulerConfiguration leafQueueConfigs, String queueName, - AbstractManagedParentQueue parent, CSQueue old) throws IOException { - super(cs, leafQueueConfigs, queueName, parent, old); + super(queueContext, queueName, parent, old); this.parent = parent; } @@ -71,7 +63,7 @@ public void setEntitlement(QueueEntitlement entitlement) @Override protected Resource getMinimumAbsoluteResource(String queuePath, String label) { - return super.getMinimumAbsoluteResource(csContext.getConfiguration() + return super.getMinimumAbsoluteResource(queueContext.getConfiguration() .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()), label); } @@ -79,7 +71,7 @@ protected Resource getMinimumAbsoluteResource(String queuePath, @Override protected Resource getMaximumAbsoluteResource(String queuePath, String label) { - return super.getMaximumAbsoluteResource(csContext.getConfiguration() + return super.getMaximumAbsoluteResource(queueContext.getConfiguration() .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()), label); } @@ -87,7 +79,7 @@ protected Resource getMaximumAbsoluteResource(String queuePath, @Override protected boolean checkConfigTypeIsAbsoluteResource(String queuePath, String label) { - return super.checkConfigTypeIsAbsoluteResource(csContext.getConfiguration() + return super.checkConfigTypeIsAbsoluteResource(queueContext.getConfiguration() .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()), label); } @@ -122,7 +114,7 @@ public void setEntitlement(String nodeLabel, QueueEntitlement entitlement) //update queue used capacity etc CSQueueUtils.updateQueueStatistics(resourceCalculator, - csContext.getClusterResource(), + queueContext.getClusterResource(), this, labelManager, nodeLabel); } finally { writeLock.unlock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 097a9dfbc57763..5040b027003d1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -117,7 +117,7 @@ public enum CapacityConfigType { private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - protected CapacitySchedulerContext csContext; + protected CapacitySchedulerQueueContext queueContext; protected YarnAuthorizationProvider authorizer = null; protected ActivitiesManager activitiesManager; @@ -131,33 +131,39 @@ public enum CapacityConfigType { // is it a dynamic queue? private boolean dynamicQueue = false; - public AbstractCSQueue(CapacitySchedulerContext cs, - String queueName, CSQueue parent, CSQueue old) throws IOException { - this(cs, cs.getConfiguration(), queueName, parent, old); - } - - public AbstractCSQueue(CapacitySchedulerContext cs, - CapacitySchedulerConfiguration configuration, String queueName, + public AbstractCSQueue(CapacitySchedulerQueueContext queueContext, String queueName, CSQueue parent, CSQueue old) { - this.labelManager = cs.getRMContext().getNodeLabelManager(); this.parent = parent; this.queuePath = createQueuePath(parent, queueName); - this.resourceCalculator = cs.getResourceCalculator(); - this.activitiesManager = cs.getActivitiesManager(); + + this.queueContext = queueContext; + this.resourceCalculator = queueContext.getResourceCalculator(); + this.activitiesManager = queueContext.getActivitiesManager(); + this.labelManager = queueContext.getLabelManager(); // must be called after parent and queueName is set CSQueueMetrics metrics = old != null ? (CSQueueMetrics) old.getMetrics() : CSQueueMetrics.forQueue(getQueuePath(), parent, - cs.getConfiguration().getEnableUserMetrics(), configuration); - usageTracker = new CSQueueUsageTracker(metrics); - this.csContext = cs; - this.queueAllocationSettings = new QueueAllocationSettings(csContext); - queueEntity = new PrivilegedEntity(EntityType.QUEUE, getQueuePath()); - queueCapacities = new QueueCapacities(parent == null); + queueContext.getConfiguration().getEnableUserMetrics(), queueContext.getConfiguration()); + this.usageTracker = new CSQueueUsageTracker(metrics); + + this.queueCapacities = new QueueCapacities(parent == null); + this.queueAllocationSettings = new QueueAllocationSettings(queueContext.getMinimumAllocation()); + + this.queueEntity = new PrivilegedEntity(EntityType.QUEUE, getQueuePath()); + + this.resourceTypes = new HashSet<>(); + for (AbsoluteResourceType type : AbsoluteResourceType.values()) { + this.resourceTypes.add(type.toString().toLowerCase()); + } + ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - readLock = lock.readLock(); - writeLock = lock.writeLock(); + this.readLock = lock.readLock(); + this.writeLock = lock.writeLock(); + + LOG.debug("Initialized {}: name={}, fullname={}", this.getClass().getSimpleName(), + queueName, getQueuePath()); } private static QueuePath createQueuePath(CSQueue parent, String queueName) { @@ -167,11 +173,6 @@ private static QueuePath createQueuePath(CSQueue parent, String queueName) { return new QueuePath(parent.getQueuePath(), queueName); } - @VisibleForTesting - protected void setupConfigurableCapacities() { - setupConfigurableCapacities(csContext.getConfiguration()); - } - protected void setupConfigurableCapacities( CapacitySchedulerConfiguration configuration) { CSQueueUtils.loadCapacitiesByLabelsFromConf(queuePath, queueCapacities, @@ -262,6 +263,10 @@ public PrivilegedEntity getPrivilegedEntity() { return queueEntity; } + public CapacitySchedulerQueueContext getQueueContext() { + return queueContext; + } + public Set getAccessibleNodeLabels() { return queueNodeLabelsSettings.getAccessibleNodeLabels(); } @@ -336,26 +341,24 @@ protected void setupQueueConfigs(Resource clusterResource, // Collect and set the Node label configuration this.queueNodeLabelsSettings = new QueueNodeLabelsSettings(configuration, parent, - getQueuePath(), csContext); + getQueuePath(), queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues()); // Initialize the queue capacities setupConfigurableCapacities(configuration); updateAbsoluteCapacities(); - updateCapacityConfigType(); // Fetch minimum/maximum resource limits for this queue if // configured - this.resourceTypes = new HashSet<>(); - for (AbsoluteResourceType type : AbsoluteResourceType.values()) { - resourceTypes.add(type.toString().toLowerCase()); - } updateConfigurableResourceLimits(clusterResource); // Setup queue's maximumAllocation respecting the global // and the queue settings - this.queueAllocationSettings.setupMaximumAllocation(configuration, getQueuePath(), - parent, csContext); + // TODO remove the getConfiguration() param after the AQC configuration duplication + // removal is resolved + this.queueAllocationSettings.setupMaximumAllocation(configuration, + queueContext.getConfiguration(), getQueuePath(), + parent); // Initialize the queue state based on previous state, configured state // and its parent state @@ -369,7 +372,8 @@ protected void setupQueueConfigs(Resource clusterResource, this.reservationsContinueLooking = configuration.getReservationContinueLook(); - this.configuredCapacityVectors = csContext.getConfiguration() + + this.configuredCapacityVectors = queueContext.getConfiguration() .parseConfiguredResourceVector(queuePath.getFullPath(), this.queueNodeLabelsSettings.getConfiguredNodeLabels()); @@ -378,7 +382,10 @@ protected void setupQueueConfigs(Resource clusterResource, this, labelManager, null); // Store preemption settings - this.preemptionSettings = new CSQueuePreemptionSettings(this, csContext, configuration); + // TODO remove the getConfiguration() param after the AQC configuration duplication + // removal is resolved + this.preemptionSettings = new CSQueuePreemptionSettings(this, configuration, + queueContext.getConfiguration()); this.priority = configuration.getQueuePriority( getQueuePath()); @@ -409,12 +416,13 @@ protected void setDynamicQueueProperties( AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX); parentTemplate = parentTemplate.substring(0, parentTemplate.lastIndexOf( DOT)); - Set parentNodeLabels = csContext - .getCapacitySchedulerQueueManager().getConfiguredNodeLabels() + Set parentNodeLabels = queueContext.getQueueManager() + .getConfiguredNodeLabelsForAllQueues() .getLabelsByQueue(parentTemplate); if (parentNodeLabels != null && parentNodeLabels.size() > 1) { - csContext.getCapacitySchedulerQueueManager().getConfiguredNodeLabels() + queueContext.getQueueManager() + .getConfiguredNodeLabelsForAllQueues() .setLabelsByQueue(getQueuePath(), new HashSet<>(parentNodeLabels)); } } @@ -436,20 +444,18 @@ private UserWeights getUserWeightsFromHierarchy( } protected Resource getMinimumAbsoluteResource(String queuePath, String label) { - Resource minResource = csContext.getConfiguration() + return queueContext.getConfiguration() .getMinimumResourceRequirement(label, queuePath, resourceTypes); - return minResource; } protected Resource getMaximumAbsoluteResource(String queuePath, String label) { - Resource maxResource = csContext.getConfiguration() + return queueContext.getConfiguration() .getMaximumResourceRequirement(label, queuePath, resourceTypes); - return maxResource; } protected boolean checkConfigTypeIsAbsoluteResource(String queuePath, String label) { - return csContext.getConfiguration().checkConfigTypeIsAbsoluteResource(label, + return queueContext.getConfiguration().checkConfigTypeIsAbsoluteResource(label, queuePath, resourceTypes); } @@ -743,7 +749,7 @@ protected void releaseResource(Resource clusterResource, } @Private - public boolean getReservationContinueLooking() { + public boolean isReservationsContinueLooking() { return reservationsContinueLooking; } @@ -764,7 +770,7 @@ public boolean getPreemptionDisabled() { @Private public boolean getIntraQueuePreemptionDisabled() { - return preemptionSettings.getIntraQueuePreemptionDisabled(); + return preemptionSettings.isIntraQueuePreemptionDisabled(); } @Private @@ -1026,12 +1032,12 @@ public Set getNodeLabelsForQueue() { } public Resource getTotalKillableResource(String partition) { - return csContext.getPreemptionManager().getKillableResource(getQueuePath(), + return queueContext.getPreemptionManager().getKillableResource(getQueuePath(), partition); } public Iterator getKillableContainers(String partition) { - return csContext.getPreemptionManager().getKillableContainers( + return queueContext.getPreemptionManager().getKillableContainers( getQueuePath(), partition); } @@ -1383,7 +1389,7 @@ public boolean isInactiveDynamicQueue() { long idleDurationSeconds = (Time.monotonicNow() - getLastSubmittedTimestamp())/1000; return isDynamicQueue() && isEligibleForAutoDeletion() && - (idleDurationSeconds > this.csContext.getConfiguration(). + (idleDurationSeconds > queueContext.getConfiguration(). getAutoExpiredDeletionTime()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java index 99911400f26e44..dff4ade9b9ef14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java @@ -19,7 +19,16 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -138,23 +147,18 @@ public class AbstractLeafQueue extends AbstractCSQueue { private final List runnableApps = new ArrayList<>(); private final List nonRunnableApps = new ArrayList<>(); - @SuppressWarnings({ "unchecked", "rawtypes" }) - public AbstractLeafQueue(CapacitySchedulerContext cs, String queueName, - CSQueue parent, CSQueue old) { - this(cs, cs.getConfiguration(), queueName, parent, old, false); + public AbstractLeafQueue(CapacitySchedulerQueueContext queueContext, + String queueName, CSQueue parent, CSQueue old) throws IOException { + this(queueContext, queueName, parent, old, false); } - public AbstractLeafQueue(CapacitySchedulerContext cs, CapacitySchedulerConfiguration configuration, - String queueName, CSQueue parent, CSQueue old) { - this(cs, configuration, queueName, parent, old, false); - } - - public AbstractLeafQueue(CapacitySchedulerContext cs, CapacitySchedulerConfiguration configuration, - String queueName, CSQueue parent, CSQueue old, boolean isDynamic) { - super(cs, configuration, queueName, parent, old); + public AbstractLeafQueue(CapacitySchedulerQueueContext queueContext, + String queueName, CSQueue parent, CSQueue old, boolean isDynamic) throws + IOException { + super(queueContext, queueName, parent, old); setDynamicQueue(isDynamic); - this.usersManager = new UsersManager(usageTracker.getMetrics(), this, labelManager, csContext, + this.usersManager = new UsersManager(usageTracker.getMetrics(), this, labelManager, resourceCalculator); // One time initialization is enough since it is static ordering policy @@ -162,16 +166,20 @@ public AbstractLeafQueue(CapacitySchedulerContext cs, CapacitySchedulerConfigura } @SuppressWarnings("checkstyle:nowhitespaceafter") - protected void setupQueueConfigs( - Resource clusterResource, CapacitySchedulerConfiguration conf) throws IOException { + protected void setupQueueConfigs(Resource clusterResource, + CapacitySchedulerConfiguration conf) throws + IOException { writeLock.lock(); try { - CapacitySchedulerConfiguration schedConf = csContext.getConfiguration(); + // TODO conf parameter can be a modified configuration with template entries and missing + // some global configs. This config duplication needs to be removed. + CapacitySchedulerConfiguration originalConfiguration = queueContext.getConfiguration(); super.setupQueueConfigs(clusterResource, conf); this.lastClusterResource = clusterResource; - this.cachedResourceLimitsForHeadroom = new ResourceLimits(clusterResource); + this.cachedResourceLimitsForHeadroom = new ResourceLimits( + clusterResource); // Initialize headroom info, also used for calculating application // master resource limits. Since this happens during queue initialization @@ -180,44 +188,52 @@ protected void setupQueueConfigs( // absoluteMaxAvailCapacity during headroom/userlimit/allocation events) setQueueResourceLimitsInfo(clusterResource); - setOrderingPolicy(conf.getAppOrderingPolicy(getQueuePath())); + setOrderingPolicy( + conf.getAppOrderingPolicy(getQueuePath())); usersManager.setUserLimit(conf.getUserLimit(getQueuePath())); usersManager.setUserLimitFactor(conf.getUserLimitFactor(getQueuePath())); maxAMResourcePerQueuePercent = - conf.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath()); + conf.getMaximumApplicationMasterResourcePerQueuePercent( + getQueuePath()); maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath()); if (maxApplications < 0) { int maxGlobalPerQueueApps = - csContext.getConfiguration().getGlobalMaximumApplicationsPerQueue(); + conf.getGlobalMaximumApplicationsPerQueue(); if (maxGlobalPerQueueApps > 0) { maxApplications = maxGlobalPerQueueApps; } } - priorityAcls = - conf.getPriorityAcls(getQueuePath(), csContext.getMaxClusterLevelAppPriority()); + priorityAcls = conf.getPriorityAcls(getQueuePath(), + originalConfiguration.getClusterLevelApplicationMaxPriority()); Set accessibleNodeLabels = this.queueNodeLabelsSettings.getAccessibleNodeLabels(); if (!SchedulerUtils.checkQueueLabelExpression(accessibleNodeLabels, this.queueNodeLabelsSettings.getDefaultLabelExpression(), null)) { - throw new IOException("Invalid default label expression of " + " queue=" + getQueuePath() - + " doesn't have permission to access all labels " - + "in default label expression. labelExpression of resource request=" - + getDefaultNodeLabelExpressionStr() + ". Queue labels=" + ( - getAccessibleNodeLabels() == null ? "" : - StringUtils.join(getAccessibleNodeLabels().iterator(), ','))); + throw new IOException( + "Invalid default label expression of " + " queue=" + getQueuePath() + + " doesn't have permission to access all labels " + + "in default label expression. labelExpression of resource request=" + + getDefaultNodeLabelExpressionStr() + ". Queue labels=" + ( + getAccessibleNodeLabels() == null ? + "" : + StringUtils + .join(getAccessibleNodeLabels().iterator(), ','))); } - nodeLocalityDelay = schedConf.getNodeLocalityDelay(); - rackLocalityAdditionalDelay = schedConf.getRackLocalityAdditionalDelay(); - rackLocalityFullReset = schedConf.getRackLocalityFullReset(); + nodeLocalityDelay = originalConfiguration.getNodeLocalityDelay(); + rackLocalityAdditionalDelay = originalConfiguration + .getRackLocalityAdditionalDelay(); + rackLocalityFullReset = originalConfiguration + .getRackLocalityFullReset(); // re-init this since max allocation could have changed this.minimumAllocationFactor = Resources.ratio(resourceCalculator, - Resources.subtract(queueAllocationSettings.getMaximumAllocation(), + Resources.subtract( + queueAllocationSettings.getMaximumAllocation(), queueAllocationSettings.getMinimumAllocation()), queueAllocationSettings.getMaximumAllocation()); @@ -233,8 +249,8 @@ protected void setupQueueConfigs( } } - defaultAppPriorityPerQueue = - Priority.newInstance(conf.getDefaultApplicationPriorityConfPerQueue(getQueuePath())); + defaultAppPriorityPerQueue = Priority.newInstance( + conf.getDefaultApplicationPriorityConfPerQueue(getQueuePath())); // Validate leaf queue's user's weights. float queueUserLimit = Math.min(100.0f, conf.getUserLimit(getQueuePath())); @@ -369,7 +385,8 @@ void setUserLimitFactor(float userLimitFactor) { public int getNumApplications() { readLock.lock(); try { - return getNumPendingApplications() + getNumActiveApplications() + getNumNonRunnableApps(); + return getNumPendingApplications() + getNumActiveApplications() + + getNumNonRunnableApps(); } finally { readLock.unlock(); } @@ -527,10 +544,11 @@ protected void reinitialize( } // Sanity check - if (!(newlyParsedQueue instanceof AbstractLeafQueue) || !newlyParsedQueue.getQueuePath() - .equals(getQueuePath())) { - throw new IOException("Trying to reinitialize " + getQueuePath() + " from " - + newlyParsedQueue.getQueuePath()); + if (!(newlyParsedQueue instanceof AbstractLeafQueue) || !newlyParsedQueue + .getQueuePath().equals(getQueuePath())) { + throw new IOException( + "Trying to reinitialize " + getQueuePath() + " from " + + newlyParsedQueue.getQueuePath()); } AbstractLeafQueue newlyParsedLeafQueue = (AbstractLeafQueue) newlyParsedQueue; @@ -556,9 +574,9 @@ protected void reinitialize( @Override public void reinitialize( CSQueue newlyParsedQueue, Resource clusterResource) - throws IOException { + throws IOException { reinitialize(newlyParsedQueue, clusterResource, - csContext.getConfiguration()); + queueContext.getConfiguration()); } @Override @@ -724,10 +742,10 @@ public Resource getUserAMResourceLimitPerPartition( Resource preWeighteduserAMLimit = Resources.multiplyAndNormalizeUp( - resourceCalculator, queuePartitionResource, - queueCapacities.getMaxAMResourcePercentage(nodePartition) + resourceCalculator, queuePartitionResource, + queueCapacities.getMaxAMResourcePercentage(nodePartition) * preWeightedUserLimit * usersManager.getUserLimitFactor(), - minimumAllocation); + minimumAllocation); if (getUserLimitFactor() == -1) { preWeighteduserAMLimit = Resources.multiplyAndNormalizeUp( @@ -752,7 +770,8 @@ public Resource getUserAMResourceLimitPerPartition( } - public Resource calculateAndGetAMResourceLimitPerPartition(String nodePartition) { + public Resource calculateAndGetAMResourceLimitPerPartition( + String nodePartition) { writeLock.lock(); try { /* @@ -769,12 +788,13 @@ public Resource calculateAndGetAMResourceLimitPerPartition(String nodePartition) // For non-labeled partition, we need to consider the current queue // usage limit. if (nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) { - synchronized (queueResourceLimitsInfo) { + synchronized (queueResourceLimitsInfo){ queueCurrentLimit = queueResourceLimitsInfo.getQueueCurrentLimit(); } } - float amResourcePercent = queueCapacities.getMaxAMResourcePercentage(nodePartition); + float amResourcePercent = queueCapacities.getMaxAMResourcePercentage( + nodePartition); // Current usable resource for this queue and partition is the max of // queueCurrentLimit and queuePartitionResource. @@ -782,13 +802,13 @@ public Resource calculateAndGetAMResourceLimitPerPartition(String nodePartition) // guarantee, use the guarantee as the queuePartitionUsableResource // because nothing less than the queue's guarantee should be used when // calculating the AM limit. - Resource queuePartitionUsableResource = - (Resources.fitsIn(resourceCalculator, queuePartitionResource, queueCurrentLimit)) ? - queueCurrentLimit : queuePartitionResource; + Resource queuePartitionUsableResource = (Resources.fitsIn( + resourceCalculator, queuePartitionResource, queueCurrentLimit)) ? + queueCurrentLimit : queuePartitionResource; - Resource amResouceLimit = - Resources.multiplyAndNormalizeUp(resourceCalculator, queuePartitionUsableResource, - amResourcePercent, queueAllocationSettings.getMinimumAllocation()); + Resource amResouceLimit = Resources.multiplyAndNormalizeUp( + resourceCalculator, queuePartitionUsableResource, amResourcePercent, + queueAllocationSettings.getMinimumAllocation()); usageTracker.getMetrics().setAMResouceLimit(nodePartition, amResouceLimit); usageTracker.getQueueUsage().setAMLimit(nodePartition, amResouceLimit); @@ -807,7 +827,8 @@ protected void activateApplications() { writeLock.lock(); try { // limit of allowed resource usage for application masters - Map userAmPartitionLimit = new HashMap(); + Map userAmPartitionLimit = + new HashMap(); // AM Resource Limit for accessible labels can be pre-calculated. // This will help in updating AMResourceLimit for all labels when queue @@ -816,8 +837,10 @@ protected void activateApplications() { calculateAndGetAMResourceLimitPerPartition(nodePartition); } - for (Iterator fsApp = getPendingAppsOrderingPolicy().getAssignmentIterator( - IteratorSelector.EMPTY_ITERATOR_SELECTOR); fsApp.hasNext(); ) { + for (Iterator fsApp = + getPendingAppsOrderingPolicy() + .getAssignmentIterator(IteratorSelector.EMPTY_ITERATOR_SELECTOR); + fsApp.hasNext(); ) { FiCaSchedulerApp application = fsApp.next(); ApplicationId applicationId = application.getApplicationId(); @@ -831,7 +854,8 @@ protected void activateApplications() { amLimit = calculateAndGetAMResourceLimitPerPartition(partitionName); } // Check am resource limit. - Resource amIfStarted = Resources.add(application.getAMResource(partitionName), + Resource amIfStarted = Resources.add( + application.getAMResource(partitionName), usageTracker.getQueueUsage().getAMUsed(partitionName)); if (LOG.isDebugEnabled()) { @@ -844,19 +868,18 @@ protected void activateApplications() { } if (!resourceCalculator.fitsIn(amIfStarted, amLimit)) { - if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(resourceCalculator, - lastClusterResource, usageTracker.getQueueUsage().getAMUsed(partitionName), - Resources.none()))) { + if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual( + resourceCalculator, lastClusterResource, + usageTracker.getQueueUsage().getAMUsed(partitionName), Resources.none()))) { LOG.warn("maximum-am-resource-percent is insufficient to start a" + " single application in queue, it is likely set too low." - + " skipping enforcement to allow at least one application" + " to start"); - } else { - application.updateAMContainerDiagnostics( - SchedulerApplicationAttempt.AMState.INACTIVATED, + + " skipping enforcement to allow at least one application" + + " to start"); + } else{ + application.updateAMContainerDiagnostics(SchedulerApplicationAttempt.AMState.INACTIVATED, CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED); - LOG.debug( - "Not activating application {} as amIfStarted: {}" + " exceeds amLimit: {}", - applicationId, amIfStarted, amLimit); + LOG.debug("Not activating application {} as amIfStarted: {}" + + " exceeds amLimit: {}", applicationId, amIfStarted, amLimit); continue; } } @@ -867,23 +890,25 @@ protected void activateApplications() { // Verify whether we already calculated user-am-limit for this label. if (userAMLimit == null) { - userAMLimit = getUserAMResourceLimitPerPartition(partitionName, application.getUser()); + userAMLimit = getUserAMResourceLimitPerPartition(partitionName, + application.getUser()); userAmPartitionLimit.put(partitionName, userAMLimit); } - Resource userAmIfStarted = Resources.add(application.getAMResource(partitionName), + Resource userAmIfStarted = Resources.add( + application.getAMResource(partitionName), user.getConsumedAMResources(partitionName)); if (!resourceCalculator.fitsIn(userAmIfStarted, userAMLimit)) { - if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(resourceCalculator, - lastClusterResource, usageTracker.getQueueUsage().getAMUsed(partitionName), - Resources.none()))) { + if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual( + resourceCalculator, lastClusterResource, + usageTracker.getQueueUsage().getAMUsed(partitionName), Resources.none()))) { LOG.warn("maximum-am-resource-percent is insufficient to start a" + " single application in queue for user, it is likely set too" - + " low. skipping enforcement to allow at least one application" + " to start"); - } else { - application.updateAMContainerDiagnostics( - AMState.INACTIVATED, + + " low. skipping enforcement to allow at least one application" + + " to start"); + } else{ + application.updateAMContainerDiagnostics(AMState.INACTIVATED, CSAMContainerLaunchDiagnosticsConstants.USER_AM_RESOURCE_LIMIT_EXCEED); LOG.debug("Not activating application {} for user: {} as" + " userAmIfStarted: {} exceeds userAmLimit: {}", @@ -893,17 +918,17 @@ protected void activateApplications() { } user.activateApplication(); orderingPolicy.addSchedulableEntity(application); - application.updateAMContainerDiagnostics(AMState.ACTIVATED, - null); + application.updateAMContainerDiagnostics(AMState.ACTIVATED, null); - usageTracker.getQueueUsage() - .incAMUsed(partitionName, application.getAMResource(partitionName)); - user.getResourceUsage().incAMUsed(partitionName, application.getAMResource(partitionName)); + usageTracker.getQueueUsage().incAMUsed(partitionName, + application.getAMResource(partitionName)); + user.getResourceUsage().incAMUsed(partitionName, + application.getAMResource(partitionName)); user.getResourceUsage().setAMLimit(partitionName, userAMLimit); usageTracker.getMetrics().incAMUsed(partitionName, application.getUser(), application.getAMResource(partitionName)); - usageTracker.getMetrics() - .setAMResouceLimitForUser(partitionName, application.getUser(), userAMLimit); + usageTracker.getMetrics().setAMResouceLimitForUser(partitionName, + application.getUser(), userAMLimit); fsApp.remove(); LOG.info("Application " + applicationId + " from user: " + application .getUser() + " activated in queue: " + getQueuePath()); @@ -913,10 +938,12 @@ protected void activateApplications() { } } - private void addApplicationAttempt(FiCaSchedulerApp application, User user) { + private void addApplicationAttempt(FiCaSchedulerApp application, + User user) { writeLock.lock(); try { - applicationAttemptMap.put(application.getApplicationAttemptId(), application); + applicationAttemptMap.put(application.getApplicationAttemptId(), + application); if (application.isRunnable()) { runnableApps.add(application); @@ -934,8 +961,8 @@ private void addApplicationAttempt(FiCaSchedulerApp application, User user) { getPendingAppsOrderingPolicy().addSchedulableEntity(application); // Activate applications - if (Resources.greaterThan(resourceCalculator, lastClusterResource, lastClusterResource, - Resources.none())) { + if (Resources.greaterThan(resourceCalculator, lastClusterResource, + lastClusterResource, Resources.none())) { activateApplications(); } else { application.updateAMContainerDiagnostics(AMState.INACTIVATED, @@ -978,7 +1005,8 @@ public void finishApplicationAttempt(FiCaSchedulerApp application, String queue) parent.finishApplicationAttempt(application, queue); } - private void removeApplicationAttempt(FiCaSchedulerApp application, String userName) { + private void removeApplicationAttempt( + FiCaSchedulerApp application, String userName) { writeLock.lock(); try { @@ -999,10 +1027,11 @@ private void removeApplicationAttempt(FiCaSchedulerApp application, String userN boolean wasActive = orderingPolicy.removeSchedulableEntity(application); if (!wasActive) { pendingOrderingPolicy.removeSchedulableEntity(application); - } else { - usageTracker.getQueueUsage() - .decAMUsed(partitionName, application.getAMResource(partitionName)); - user.getResourceUsage().decAMUsed(partitionName, application.getAMResource(partitionName)); + } else{ + usageTracker.getQueueUsage().decAMUsed(partitionName, + application.getAMResource(partitionName)); + user.getResourceUsage().decAMUsed(partitionName, + application.getAMResource(partitionName)); usageTracker.getMetrics().decAMUsed(partitionName, application.getUser(), application.getAMResource(partitionName)); } @@ -1028,7 +1057,8 @@ private void removeApplicationAttempt(FiCaSchedulerApp application, String userN } } - private FiCaSchedulerApp getApplication(ApplicationAttemptId applicationAttemptId) { + private FiCaSchedulerApp getApplication( + ApplicationAttemptId applicationAttemptId) { return applicationAttemptMap.get(applicationAttemptId); } @@ -1038,7 +1068,7 @@ private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) { if (!usageTracker.getQueueResourceQuotas().getEffectiveMinResource(nodePartition) .equals(Resources.none())) { limits.setIsAllowPreemption(Resources.lessThan(resourceCalculator, - csContext.getClusterResource(), usageTracker.getQueueUsage().getUsed(nodePartition), + queueContext.getClusterResource(), usageTracker.getQueueUsage().getUsed(nodePartition), usageTracker.getQueueResourceQuotas().getEffectiveMinResource(nodePartition))); return; } @@ -1049,8 +1079,8 @@ private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) { } private CSAssignment allocateFromReservedContainer(Resource clusterResource, - CandidateNodeSet candidates, ResourceLimits currentResourceLimits, - SchedulingMode schedulingMode) { + CandidateNodeSet candidates, + ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) { // Irrespective of Single / Multi Node Placement, the allocate from // Reserved Container has to happen only for the single node which @@ -1062,14 +1092,15 @@ private CSAssignment allocateFromReservedContainer(Resource clusterResource, if (node != null) { RMContainer reservedContainer = node.getReservedContainer(); if (reservedContainer != null) { - FiCaSchedulerApp application = getApplication(reservedContainer.getApplicationAttemptId()); + FiCaSchedulerApp application = getApplication( + reservedContainer.getApplicationAttemptId()); if (null != application) { - ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager, node, - SystemClock.getInstance().getTime(), application); - CSAssignment assignment = - application.assignContainers(clusterResource, candidates, currentResourceLimits, - schedulingMode, reservedContainer); + ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager, + node, SystemClock.getInstance().getTime(), application); + CSAssignment assignment = application.assignContainers( + clusterResource, candidates, currentResourceLimits, + schedulingMode, reservedContainer); return assignment; } } @@ -1078,7 +1109,8 @@ private CSAssignment allocateFromReservedContainer(Resource clusterResource, return null; } - private ConcurrentMap getUserLimitCache(String partition, + private ConcurrentMap getUserLimitCache( + String partition, SchedulingMode schedulingMode) { synchronized (userLimitsCache) { long latestVersion = usersManager.getLatestVersionOfUsersState(); @@ -1088,19 +1120,20 @@ private ConcurrentMap getUserLimitCache(String partitio this.currentUserLimitCacheVersion = latestVersion; userLimitsCache.clear(); - Map> uLCByPartition = - new HashMap<>(); + Map> + uLCByPartition = new HashMap<>(); userLimitsCache.put(partition, uLCByPartition); - ConcurrentMap uLCBySchedulingMode = new ConcurrentHashMap<>(); + ConcurrentMap uLCBySchedulingMode = + new ConcurrentHashMap<>(); uLCByPartition.put(schedulingMode, uLCBySchedulingMode); return uLCBySchedulingMode; } // User limits cache does not need invalidating - Map> uLCByPartition = - userLimitsCache.get(partition); + Map> + uLCByPartition = userLimitsCache.get(partition); if (uLCByPartition == null) { uLCByPartition = new HashMap<>(); userLimitsCache.put(partition, uLCByPartition); @@ -1119,8 +1152,8 @@ private ConcurrentMap getUserLimitCache(String partitio @Override public CSAssignment assignContainers(Resource clusterResource, - CandidateNodeSet candidates, ResourceLimits currentResourceLimits, - SchedulingMode schedulingMode) { + CandidateNodeSet candidates, + ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) { updateCurrentResourceLimits(currentResourceLimits, clusterResource); FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates); @@ -1132,9 +1165,8 @@ public CSAssignment assignContainers(Resource clusterResource, setPreemptionAllowed(currentResourceLimits, candidates.getPartition()); // Check for reserved resources, try to allocate reserved container first. - CSAssignment assignment = - allocateFromReservedContainer(clusterResource, candidates, currentResourceLimits, - schedulingMode); + CSAssignment assignment = allocateFromReservedContainer(clusterResource, + candidates, currentResourceLimits, schedulingMode); if (null != assignment) { return assignment; } @@ -1169,29 +1201,33 @@ public CSAssignment assignContainers(Resource clusterResource, boolean needAssignToQueueCheck = true; IteratorSelector sel = new IteratorSelector(); sel.setPartition(candidates.getPartition()); - for (Iterator assignmentIterator = orderingPolicy.getAssignmentIterator(sel); + for (Iterator assignmentIterator = + orderingPolicy.getAssignmentIterator(sel); assignmentIterator.hasNext(); ) { FiCaSchedulerApp application = assignmentIterator.next(); - ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager, node, - SystemClock.getInstance().getTime(), application); + ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager, + node, SystemClock.getInstance().getTime(), application); // Check queue max-capacity limit Resource appReserved = application.getCurrentReservation(); if (needAssignToQueueCheck) { - if (!super.canAssignToThisQueue(clusterResource, candidates.getPartition(), - currentResourceLimits, appReserved, schedulingMode)) { - ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(activitiesManager, node, - application, application.getPriority(), + if (!super.canAssignToThisQueue(clusterResource, + candidates.getPartition(), currentResourceLimits, appReserved, + schedulingMode)) { + ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue( + activitiesManager, node, application, application.getPriority(), ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, parent.getQueuePath(), - getQueuePath(), ActivityState.REJECTED, + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, + parent.getQueuePath(), getQueuePath(), + ActivityState.REJECTED, ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); return CSAssignment.NULL_ASSIGNMENT; } // If there was no reservation and canAssignToThisQueue returned // true, there is no reason to check further. - if (!this.reservationsContinueLooking || appReserved.equals(Resources.none())) { + if (!this.reservationsContinueLooking + || appReserved.equals(Resources.none())) { needAssignToQueueCheck = false; } } @@ -1201,12 +1237,13 @@ public CSAssignment assignContainers(Resource clusterResource, if (cul != null) { cachedUserLimit = cul.userLimit; } - Resource userLimit = - computeUserLimitAndSetHeadroom(application, clusterResource, candidates.getPartition(), - schedulingMode, cachedUserLimit); + Resource userLimit = computeUserLimitAndSetHeadroom(application, + clusterResource, candidates.getPartition(), schedulingMode, + cachedUserLimit); if (cul == null) { cul = new CachedUserLimit(userLimit); - CachedUserLimit retVal = userLimits.putIfAbsent(application.getUser(), cul); + CachedUserLimit retVal = + userLimits.putIfAbsent(application.getUser(), cul); if (retVal != null) { // another thread updated the user limit cache before us cul = retVal; @@ -1218,9 +1255,9 @@ public CSAssignment assignContainers(Resource clusterResource, if (!cul.canAssign && Resources.fitsIn(appReserved, cul.reservation)) { userAssignable = false; } else { - userAssignable = - canAssignToUser(clusterResource, application.getUser(), userLimit, application, - candidates.getPartition(), currentResourceLimits); + userAssignable = canAssignToUser(clusterResource, application.getUser(), + userLimit, application, candidates.getPartition(), + currentResourceLimits); if (!userAssignable && Resources.fitsIn(cul.reservation, appReserved)) { cul.canAssign = false; cul.reservation = appReserved; @@ -1229,54 +1266,59 @@ public CSAssignment assignContainers(Resource clusterResource, if (!userAssignable) { application.updateAMContainerDiagnostics(AMState.ACTIVATED, "User capacity has reached its maximum limit."); - ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(activitiesManager, node, - application, application.getPriority(), + ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue( + activitiesManager, node, application, application.getPriority(), ActivityDiagnosticConstant.QUEUE_HIT_USER_MAX_CAPACITY_LIMIT); continue; } // Try to schedule - assignment = application.assignContainers(clusterResource, candidates, currentResourceLimits, - schedulingMode, null); + assignment = application.assignContainers(clusterResource, + candidates, currentResourceLimits, schedulingMode, null); if (LOG.isDebugEnabled()) { - LOG.debug( - "post-assignContainers for application " + application.getApplicationId()); + LOG.debug("post-assignContainers for application " + application + .getApplicationId()); application.showRequests(); } // Did we schedule or reserve a container? Resource assigned = assignment.getResource(); - if (Resources.greaterThan(resourceCalculator, clusterResource, assigned, Resources.none())) { - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, parent.getQueuePath(), - getQueuePath(), ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); + if (Resources.greaterThan(resourceCalculator, clusterResource, assigned, + Resources.none())) { + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, + parent.getQueuePath(), getQueuePath(), + ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); return assignment; - } else if (assignment.getSkippedType() == CSAssignment.SkippedType.OTHER) { - ActivitiesLogger.APP.finishSkippedAppAllocationRecording(activitiesManager, - application.getApplicationId(), ActivityState.SKIPPED, - ActivityDiagnosticConstant.EMPTY); + } else if (assignment.getSkippedType() + == CSAssignment.SkippedType.OTHER) { + ActivitiesLogger.APP.finishSkippedAppAllocationRecording( + activitiesManager, application.getApplicationId(), + ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY); application.updateNodeInfoForAMDiagnostics(node); - } else if (assignment.getSkippedType() == CSAssignment.SkippedType.QUEUE_LIMIT) { - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, parent.getQueuePath(), - getQueuePath(), ActivityState.REJECTED, - () -> ActivityDiagnosticConstant.QUEUE_DO_NOT_HAVE_ENOUGH_HEADROOM + " from " - + application.getApplicationId()); + } else if (assignment.getSkippedType() + == CSAssignment.SkippedType.QUEUE_LIMIT) { + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, + parent.getQueuePath(), getQueuePath(), ActivityState.REJECTED, + () -> ActivityDiagnosticConstant.QUEUE_DO_NOT_HAVE_ENOUGH_HEADROOM + + " from " + application.getApplicationId()); return assignment; - } else { + } else{ // If we don't allocate anything, and it is not skipped by application, // we will return to respect FIFO of applications - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, parent.getQueuePath(), - getQueuePath(), ActivityState.SKIPPED, + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, + parent.getQueuePath(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.QUEUE_SKIPPED_TO_RESPECT_FIFO); - ActivitiesLogger.APP.finishSkippedAppAllocationRecording(activitiesManager, - application.getApplicationId(), ActivityState.SKIPPED, - ActivityDiagnosticConstant.EMPTY); + ActivitiesLogger.APP.finishSkippedAppAllocationRecording( + activitiesManager, application.getApplicationId(), + ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY); return CSAssignment.NULL_ASSIGNMENT; } } - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, parent.getQueuePath(), - getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY); + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, + parent.getQueuePath(), getQueuePath(), ActivityState.SKIPPED, + ActivityDiagnosticConstant.EMPTY); return CSAssignment.NULL_ASSIGNMENT; } @@ -1293,13 +1335,14 @@ public boolean accept(Resource cluster, if (allocation.getAllocateFromReservedContainer() == null) { readLock.lock(); try { - FiCaSchedulerApp app = schedulerContainer.getSchedulerApplicationAttempt(); + FiCaSchedulerApp app = + schedulerContainer.getSchedulerApplicationAttempt(); String username = app.getUser(); String p = schedulerContainer.getNodePartition(); // check user-limit - Resource userLimit = - computeUserLimitAndSetHeadroom(app, cluster, p, allocation.getSchedulingMode(), null); + Resource userLimit = computeUserLimitAndSetHeadroom(app, cluster, p, + allocation.getSchedulingMode(), null); // Deduct resources that we can release User user = getUser(username); @@ -1308,11 +1351,13 @@ public boolean accept(Resource cluster, return false; } Resource usedResource = Resources.clone(user.getUsed(p)); - Resources.subtractFrom(usedResource, request.getTotalReleasedResource()); + Resources.subtractFrom(usedResource, + request.getTotalReleasedResource()); - if (Resources.greaterThan(resourceCalculator, cluster, usedResource, userLimit)) { - LOG.debug("Used resource={} exceeded user-limit={}", usedResource, - userLimit); + if (Resources.greaterThan(resourceCalculator, cluster, usedResource, + userLimit)) { + LOG.debug("Used resource={} exceeded user-limit={}", + usedResource, userLimit); return false; } } finally { @@ -1335,34 +1380,42 @@ private void internalReleaseContainer(Resource clusterResource, if (rmContainer.getState() == RMContainerState.RESERVED) { // For other reserved containers // This is a reservation exchange, complete previous reserved container - completedContainer(clusterResource, schedulerContainer.getSchedulerApplicationAttempt(), - schedulerContainer.getSchedulerNode(), rmContainer, - SchedulerUtils.createAbnormalContainerStatus(rmContainer.getContainerId(), - SchedulerUtils.UNRESERVED_CONTAINER), RMContainerEventType.RELEASED, null, false); + completedContainer(clusterResource, + schedulerContainer.getSchedulerApplicationAttempt(), + schedulerContainer.getSchedulerNode(), rmContainer, SchedulerUtils + .createAbnormalContainerStatus(rmContainer.getContainerId(), + SchedulerUtils.UNRESERVED_CONTAINER), + RMContainerEventType.RELEASED, null, false); } - } else { + } else{ // When trying to preempt containers from different queue -- this // is for lazy preemption feature (kill preemption candidate in scheduling // cycle). targetLeafQueue.completedContainer(clusterResource, schedulerContainer.getSchedulerApplicationAttempt(), - schedulerContainer.getSchedulerNode(), schedulerContainer.getRmContainer(), - SchedulerUtils.createPreemptedContainerStatus(rmContainer.getContainerId(), - SchedulerUtils.PREEMPTED_CONTAINER), RMContainerEventType.KILL, null, false); + schedulerContainer.getSchedulerNode(), + schedulerContainer.getRmContainer(), SchedulerUtils + .createPreemptedContainerStatus(rmContainer.getContainerId(), + SchedulerUtils.PREEMPTED_CONTAINER), + RMContainerEventType.KILL, null, false); } } private void releaseContainers(Resource clusterResource, ResourceCommitRequest request) { - for (SchedulerContainer c : request.getContainersToRelease()) { + for (SchedulerContainer c : request + .getContainersToRelease()) { internalReleaseContainer(clusterResource, c); } // Handle container reservation looking, or lazy preemption case: - if (null != request.getContainersToAllocate() && !request.getContainersToAllocate().isEmpty()) { - for (ContainerAllocationProposal context : request.getContainersToAllocate()) { + if (null != request.getContainersToAllocate() && !request + .getContainersToAllocate().isEmpty()) { + for (ContainerAllocationProposal context : request + .getContainersToAllocate()) { if (null != context.getToRelease()) { - for (SchedulerContainer c : context.getToRelease()) { + for (SchedulerContainer c : context + .getToRelease()) { internalReleaseContainer(clusterResource, c); } } @@ -1380,10 +1433,10 @@ public void apply(Resource cluster, writeLock.lock(); try { if (request.anythingAllocatedOrReserved()) { - ContainerAllocationProposal allocation = - request.getFirstAllocatedOrReservedContainer(); - SchedulerContainer schedulerContainer = - allocation.getAllocatedOrReservedContainer(); + ContainerAllocationProposal + allocation = request.getFirstAllocatedOrReservedContainer(); + SchedulerContainer + schedulerContainer = allocation.getAllocatedOrReservedContainer(); // Do not modify queue when allocation from reserved container if (allocation.getAllocateFromReservedContainer() == null) { @@ -1392,16 +1445,19 @@ public void apply(Resource cluster, applyToParentQueue = true; // Book-keeping // Note: Update headroom to account for current allocation too... - allocateResource(cluster, schedulerContainer.getSchedulerApplicationAttempt(), - allocation.getAllocatedOrReservedResource(), schedulerContainer.getNodePartition(), + allocateResource(cluster, + schedulerContainer.getSchedulerApplicationAttempt(), + allocation.getAllocatedOrReservedResource(), + schedulerContainer.getNodePartition(), schedulerContainer.getRmContainer()); - orderingPolicy.containerAllocated(schedulerContainer.getSchedulerApplicationAttempt(), + orderingPolicy.containerAllocated( + schedulerContainer.getSchedulerApplicationAttempt(), schedulerContainer.getRmContainer()); } // Update reserved resource - if (Resources.greaterThan(resourceCalculator, cluster, request.getTotalReservedResource(), - Resources.none())) { + if (Resources.greaterThan(resourceCalculator, cluster, + request.getTotalReservedResource(), Resources.none())) { incReservedResource(schedulerContainer.getNodePartition(), request.getTotalReservedResource()); } @@ -1415,6 +1471,7 @@ public void apply(Resource cluster, } } + protected Resource getHeadroom(User user, Resource queueCurrentLimit, Resource clusterResource, FiCaSchedulerApp application) { return getHeadroom(user, queueCurrentLimit, clusterResource, application, @@ -1425,8 +1482,9 @@ protected Resource getHeadroom(User user, Resource queueCurrentLimit, Resource clusterResource, FiCaSchedulerApp application, String partition) { return getHeadroom(user, queueCurrentLimit, clusterResource, - getResourceLimitForActiveUsers(application.getUser(), clusterResource, partition, - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), partition); + getResourceLimitForActiveUsers(application.getUser(), clusterResource, + partition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), + partition); } private Resource getHeadroom(User user, @@ -1455,30 +1513,36 @@ private Resource getHeadroom(User user, * headroom = min (unused resourcelimit of a label, calculated headroom ) */ currentPartitionResourceLimit = - partition.equals(RMNodeLabelsManager.NO_LABEL) ? currentPartitionResourceLimit : - getQueueMaxResource(partition); + partition.equals(RMNodeLabelsManager.NO_LABEL) + ? currentPartitionResourceLimit + : getQueueMaxResource(partition); Resource headroom = Resources.componentwiseMin( - Resources.subtractNonNegative(userLimitResource, user.getUsed(partition)), + Resources.subtractNonNegative(userLimitResource, + user.getUsed(partition)), Resources.subtractNonNegative(currentPartitionResourceLimit, usageTracker.getQueueUsage().getUsed(partition))); // Normalize it before return - headroom = Resources.roundDown(resourceCalculator, headroom, - queueAllocationSettings.getMinimumAllocation()); + headroom = + Resources.roundDown(resourceCalculator, headroom, + queueAllocationSettings.getMinimumAllocation()); //headroom = min (unused resourcelimit of a label, calculated headroom ) - Resource clusterPartitionResource = labelManager.getResourceByLabel(partition, clusterResource); - Resource clusterFreePartitionResource = Resources.subtract(clusterPartitionResource, - csContext.getClusterResourceUsage().getUsed(partition)); - headroom = - Resources.min(resourceCalculator, clusterPartitionResource, clusterFreePartitionResource, - headroom); + Resource clusterPartitionResource = + labelManager.getResourceByLabel(partition, clusterResource); + Resource clusterFreePartitionResource = + Resources.subtract(clusterPartitionResource, + queueContext.getClusterResourceUsage().getUsed(partition)); + headroom = Resources.min(resourceCalculator, clusterPartitionResource, + clusterFreePartitionResource, headroom); return headroom; } - private void setQueueResourceLimitsInfo(Resource clusterResource) { + private void setQueueResourceLimitsInfo( + Resource clusterResource) { synchronized (queueResourceLimitsInfo) { - queueResourceLimitsInfo.setQueueCurrentLimit(cachedResourceLimitsForHeadroom.getLimit()); + queueResourceLimitsInfo.setQueueCurrentLimit(cachedResourceLimitsForHeadroom + .getLimit()); queueResourceLimitsInfo.setClusterResource(clusterResource); } } @@ -1486,8 +1550,8 @@ private void setQueueResourceLimitsInfo(Resource clusterResource) { // It doesn't necessarily to hold application's lock here. @Lock({AbstractLeafQueue.class}) Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application, - Resource clusterResource, String nodePartition, SchedulingMode schedulingMode, - Resource userLimit) { + Resource clusterResource, String nodePartition, + SchedulingMode schedulingMode, Resource userLimit) { String user = application.getUser(); User queueUser = getUser(user); if (queueUser == null) { @@ -1498,25 +1562,26 @@ Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application, // Compute user limit respect requested labels, // TODO, need consider headroom respect labels also if (userLimit == null) { - userLimit = - getResourceLimitForActiveUsers(application.getUser(), clusterResource, nodePartition, - schedulingMode); + userLimit = getResourceLimitForActiveUsers(application.getUser(), + clusterResource, nodePartition, schedulingMode); } setQueueResourceLimitsInfo(clusterResource); - Resource headroom = usageTracker.getMetrics().getUserMetrics(user) == null ? Resources.none() : - getHeadroom(queueUser, cachedResourceLimitsForHeadroom.getLimit(), clusterResource, - userLimit, nodePartition); + Resource headroom = + usageTracker.getMetrics().getUserMetrics(user) == null ? Resources.none() : + getHeadroom(queueUser, cachedResourceLimitsForHeadroom.getLimit(), + clusterResource, userLimit, nodePartition); if (LOG.isDebugEnabled()) { - LOG.debug( - "Headroom calculation for user " + user + ": " + " userLimit=" + userLimit - + " queueMaxAvailRes=" + cachedResourceLimitsForHeadroom.getLimit() + " consumed=" - + queueUser.getUsed() + " partition=" + nodePartition); + LOG.debug("Headroom calculation for user " + user + ": " + " userLimit=" + + userLimit + " queueMaxAvailRes=" + + cachedResourceLimitsForHeadroom.getLimit() + " consumed=" + + queueUser.getUsed() + " partition=" + + nodePartition); } - CapacityHeadroomProvider headroomProvider = - new CapacityHeadroomProvider(queueUser, this, application, queueResourceLimitsInfo); + CapacityHeadroomProvider headroomProvider = new CapacityHeadroomProvider( + queueUser, this, application, queueResourceLimitsInfo); application.setHeadroomProvider(headroomProvider); @@ -1553,10 +1618,11 @@ public boolean getRackLocalityFullReset() { * RESPECT_PARTITION_EXCLUSIVITY/IGNORE_PARTITION_EXCLUSIVITY * @return Computed User Limit */ - public Resource getResourceLimitForActiveUsers(String userName, Resource clusterResource, - String nodePartition, SchedulingMode schedulingMode) { - return usersManager.getComputedResourceLimitForActiveUsers(userName, clusterResource, - nodePartition, schedulingMode); + public Resource getResourceLimitForActiveUsers(String userName, + Resource clusterResource, String nodePartition, + SchedulingMode schedulingMode) { + return usersManager.getComputedResourceLimitForActiveUsers(userName, + clusterResource, nodePartition, schedulingMode); } /** @@ -1572,10 +1638,11 @@ public Resource getResourceLimitForActiveUsers(String userName, Resource cluster * RESPECT_PARTITION_EXCLUSIVITY/IGNORE_PARTITION_EXCLUSIVITY * @return Computed User Limit */ - public Resource getResourceLimitForAllUsers(String userName, Resource clusterResource, - String nodePartition, SchedulingMode schedulingMode) { - return usersManager.getComputedResourceLimitForAllUsers(userName, clusterResource, - nodePartition, schedulingMode); + public Resource getResourceLimitForAllUsers(String userName, + Resource clusterResource, String nodePartition, + SchedulingMode schedulingMode) { + return usersManager.getComputedResourceLimitForAllUsers(userName, + clusterResource, nodePartition, schedulingMode); } @Private @@ -1595,24 +1662,27 @@ protected boolean canAssignToUser(Resource clusterResource, // Note: We aren't considering the current request since there is a fixed // overhead of the AM, but it's a > check, not a >= check, so... - if (Resources.greaterThan(resourceCalculator, clusterResource, user.getUsed(nodePartition), - limit)) { + if (Resources.greaterThan(resourceCalculator, clusterResource, + user.getUsed(nodePartition), limit)) { // if enabled, check to see if could we potentially use this node instead // of a reserved node if the application has reserved containers if (this.reservationsContinueLooking) { if (Resources.lessThanOrEqual(resourceCalculator, clusterResource, - Resources.subtract(user.getUsed(), application.getCurrentReservation()), limit)) { + Resources.subtract(user.getUsed(), + application.getCurrentReservation()), limit)) { if (LOG.isDebugEnabled()) { LOG.debug("User " + userName + " in queue " + getQueuePath() - + " will exceed limit based on reservations - " + " consumed: " + user.getUsed() - + " reserved: " + application.getCurrentReservation() + " limit: " + limit); + + " will exceed limit based on reservations - " + + " consumed: " + user.getUsed() + " reserved: " + application + .getCurrentReservation() + " limit: " + limit); } - Resource amountNeededToUnreserve = - Resources.subtract(user.getUsed(nodePartition), limit); + Resource amountNeededToUnreserve = Resources.subtract( + user.getUsed(nodePartition), limit); // we can only acquire a new container if we unreserve first to // respect user-limit - currentResourceLimits.setAmountNeededUnreserve(amountNeededToUnreserve); + currentResourceLimits.setAmountNeededUnreserve( + amountNeededToUnreserve); return true; } } @@ -1639,10 +1709,10 @@ protected void setDynamicQueueProperties(CapacitySchedulerConfiguration configur super.setDynamicQueueProperties(configuration); } - private void updateSchedulerHealthForCompletedContainer(RMContainer rmContainer, - ContainerStatus containerStatus) { + private void updateSchedulerHealthForCompletedContainer( + RMContainer rmContainer, ContainerStatus containerStatus) { // Update SchedulerHealth for released / preempted container - SchedulerHealth schedulerHealth = csContext.getSchedulerHealth(); + SchedulerHealth schedulerHealth = queueContext.getSchedulerHealth(); if (null == schedulerHealth) { // Only do update if we have schedulerHealth return; @@ -1653,8 +1723,9 @@ private void updateSchedulerHealthForCompletedContainer(RMContainer rmContainer, rmContainer.getContainerId(), getQueuePath()); schedulerHealth.updateSchedulerPreemptionCounts(1); } else { - schedulerHealth.updateRelease(csContext.getLastNodeUpdateTime(), - rmContainer.getAllocatedNode(), rmContainer.getContainerId(), getQueuePath()); + schedulerHealth.updateRelease(queueContext.getLastNodeUpdateTime(), + rmContainer.getAllocatedNode(), rmContainer.getContainerId(), + getQueuePath()); } } @@ -1666,13 +1737,15 @@ private void updateSchedulerHealthForCompletedContainer(RMContainer rmContainer, * @param nodePartition * Partition */ - public void recalculateQueueUsageRatio(Resource clusterResource, String nodePartition) { + public void recalculateQueueUsageRatio(Resource clusterResource, + String nodePartition) { writeLock.lock(); try { ResourceUsage queueResourceUsage = getQueueResourceUsage(); if (nodePartition == null) { - for (String partition : Sets.union(getQueueCapacities().getNodePartitionsSet(), + for (String partition : Sets.union( + getQueueCapacities().getNodePartitionsSet(), queueResourceUsage.getNodePartitionsSet())) { usersManager.updateUsageRatio(partition, clusterResource); } @@ -1685,9 +1758,10 @@ public void recalculateQueueUsageRatio(Resource clusterResource, String nodePart } @Override - public void completedContainer(Resource clusterResource, FiCaSchedulerApp application, - FiCaSchedulerNode node, RMContainer rmContainer, ContainerStatus containerStatus, - RMContainerEventType event, CSQueue childQueue, boolean sortQueues) { + public void completedContainer(Resource clusterResource, + FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer, + ContainerStatus containerStatus, RMContainerEventType event, CSQueue childQueue, + boolean sortQueues) { // Update SchedulerHealth for released / preempted container updateSchedulerHealthForCompletedContainer(rmContainer, containerStatus); @@ -1704,10 +1778,11 @@ public void completedContainer(Resource clusterResource, FiCaSchedulerApp applic // happen under scheduler's lock... // So, this is, in effect, a transaction across application & node if (rmContainer.getState() == RMContainerState.RESERVED) { - removed = application.unreserve(rmContainer.getReservedSchedulerKey(), node, rmContainer); - } else { - removed = application.containerCompleted(rmContainer, containerStatus, event, - node.getPartition()); + removed = application.unreserve(rmContainer.getReservedSchedulerKey(), + node, rmContainer); + } else{ + removed = application.containerCompleted(rmContainer, containerStatus, + event, node.getPartition()); node.releaseContainer(rmContainer.getContainerId(), false); } @@ -1725,16 +1800,20 @@ public void completedContainer(Resource clusterResource, FiCaSchedulerApp applic writeLock.unlock(); } + if (removed) { // Inform the parent queue _outside_ of the leaf-queue lock - parent.completedContainer(clusterResource, application, node, rmContainer, null, event, - this, sortQueues); + parent.completedContainer(clusterResource, application, node, + rmContainer, null, event, this, sortQueues); } } // Notify PreemptionManager - csContext.getPreemptionManager().removeKillableContainer( - new KillableContainer(rmContainer, node.getPartition(), getQueuePath())); + queueContext.getPreemptionManager().removeKillableContainer( + new KillableContainer( + rmContainer, + node.getPartition(), + getQueuePath())); // Update preemption metrics if exit status is PREEMPTED if (containerStatus != null @@ -1743,20 +1822,23 @@ public void completedContainer(Resource clusterResource, FiCaSchedulerApp applic } } - void allocateResource(Resource clusterResource, SchedulerApplicationAttempt application, - Resource resource, String nodePartition, RMContainer rmContainer) { + void allocateResource(Resource clusterResource, + SchedulerApplicationAttempt application, Resource resource, + String nodePartition, RMContainer rmContainer) { writeLock.lock(); try { super.allocateResource(clusterResource, resource, nodePartition); // handle ignore exclusivity container - if (null != rmContainer && rmContainer.getNodeLabelExpression() - .equals(RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals( + if (null != rmContainer && rmContainer.getNodeLabelExpression().equals( + RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals( RMNodeLabelsManager.NO_LABEL)) { TreeSet rmContainers = null; - if (null == (rmContainers = ignorePartitionExclusivityRMContainers.get(nodePartition))) { + if (null == (rmContainers = ignorePartitionExclusivityRMContainers.get( + nodePartition))) { rmContainers = new TreeSet<>(); - ignorePartitionExclusivityRMContainers.put(nodePartition, rmContainers); + ignorePartitionExclusivityRMContainers.put(nodePartition, + rmContainers); } rmContainers.add(rmContainer); } @@ -1766,17 +1848,18 @@ void allocateResource(Resource clusterResource, SchedulerApplicationAttempt appl // Increment user's resource usage. User user = usersManager.updateUserResourceUsage(userName, resource, - nodePartition, true); + queueContext.getClusterResource(), nodePartition, true); Resource partitionHeadroom = Resources.createResource(0, 0); if (usageTracker.getMetrics().getUserMetrics(userName) != null) { - partitionHeadroom = - getHeadroom(user, cachedResourceLimitsForHeadroom.getLimit(), clusterResource, - getResourceLimitForActiveUsers(userName, clusterResource, nodePartition, - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodePartition); + partitionHeadroom = getHeadroom(user, + cachedResourceLimitsForHeadroom.getLimit(), clusterResource, + getResourceLimitForActiveUsers(userName, clusterResource, + nodePartition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), + nodePartition); } - usageTracker.getMetrics() - .setAvailableResourcesToUser(nodePartition, userName, partitionHeadroom); + usageTracker.getMetrics().setAvailableResourcesToUser(nodePartition, userName, + partitionHeadroom); if (LOG.isDebugEnabled()) { LOG.debug(getQueuePath() + " user=" + userName + " used=" @@ -1789,18 +1872,20 @@ void allocateResource(Resource clusterResource, SchedulerApplicationAttempt appl } } - void releaseResource(Resource clusterResource, FiCaSchedulerApp application, Resource resource, - String nodePartition, RMContainer rmContainer) { + void releaseResource(Resource clusterResource, + FiCaSchedulerApp application, Resource resource, String nodePartition, + RMContainer rmContainer) { writeLock.lock(); try { super.releaseResource(clusterResource, resource, nodePartition); // handle ignore exclusivity container - if (null != rmContainer && rmContainer.getNodeLabelExpression() - .equals(RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals( + if (null != rmContainer && rmContainer.getNodeLabelExpression().equals( + RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals( RMNodeLabelsManager.NO_LABEL)) { if (ignorePartitionExclusivityRMContainers.containsKey(nodePartition)) { - Set rmContainers = ignorePartitionExclusivityRMContainers.get(nodePartition); + Set rmContainers = + ignorePartitionExclusivityRMContainers.get(nodePartition); rmContainers.remove(rmContainer); if (rmContainers.isEmpty()) { ignorePartitionExclusivityRMContainers.remove(nodePartition); @@ -1811,17 +1896,18 @@ void releaseResource(Resource clusterResource, FiCaSchedulerApp application, Res // Update user metrics String userName = application.getUser(); User user = usersManager.updateUserResourceUsage(userName, resource, - nodePartition, false); + queueContext.getClusterResource(), nodePartition, false); Resource partitionHeadroom = Resources.createResource(0, 0); if (usageTracker.getMetrics().getUserMetrics(userName) != null) { - partitionHeadroom = - getHeadroom(user, cachedResourceLimitsForHeadroom.getLimit(), clusterResource, - getResourceLimitForActiveUsers(userName, clusterResource, nodePartition, - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodePartition); + partitionHeadroom = getHeadroom(user, + cachedResourceLimitsForHeadroom.getLimit(), clusterResource, + getResourceLimitForActiveUsers(userName, clusterResource, + nodePartition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), + nodePartition); } - usageTracker.getMetrics() - .setAvailableResourcesToUser(nodePartition, userName, partitionHeadroom); + usageTracker.getMetrics().setAvailableResourcesToUser(nodePartition, userName, + partitionHeadroom); if (LOG.isDebugEnabled()) { LOG.debug( @@ -1834,19 +1920,20 @@ void releaseResource(Resource clusterResource, FiCaSchedulerApp application, Res } } - private void updateCurrentResourceLimits(ResourceLimits currentResourceLimits, - Resource clusterResource) { + private void updateCurrentResourceLimits( + ResourceLimits currentResourceLimits, Resource clusterResource) { // TODO: need consider non-empty node labels when resource limits supports // node labels // Even if ParentQueue will set limits respect child's max queue capacity, // but when allocating reserved container, CapacityScheduler doesn't do // this. So need cap limits by queue's max capacity here. - this.cachedResourceLimitsForHeadroom = new ResourceLimits(currentResourceLimits.getLimit()); - Resource queueMaxResource = getEffectiveMaxCapacityDown(RMNodeLabelsManager.NO_LABEL, - queueAllocationSettings.getMinimumAllocation()); - this.cachedResourceLimitsForHeadroom.setLimit( - Resources.min(resourceCalculator, clusterResource, queueMaxResource, - currentResourceLimits.getLimit())); + this.cachedResourceLimitsForHeadroom = + new ResourceLimits(currentResourceLimits.getLimit()); + Resource queueMaxResource = getEffectiveMaxCapacityDown( + RMNodeLabelsManager.NO_LABEL, queueAllocationSettings.getMinimumAllocation()); + this.cachedResourceLimitsForHeadroom.setLimit(Resources.min( + resourceCalculator, clusterResource, queueMaxResource, + currentResourceLimits.getLimit())); } @Override @@ -1861,7 +1948,7 @@ public void updateClusterResource(Resource clusterResource, super.updateEffectiveResources(clusterResource); // Update maximum applications for the queue and for users - updateMaximumApplications(csContext.getConfiguration()); + updateMaximumApplications(queueContext.getConfiguration()); updateCurrentResourceLimits(currentResourceLimits, clusterResource); @@ -1874,12 +1961,12 @@ public void updateClusterResource(Resource clusterResource, recalculateQueueUsageRatio(clusterResource, null); // Update metrics - CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, this, labelManager, - null); + CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, + this, labelManager, null); // Update configured capacity/max-capacity for default partition only CSQueueUtils.updateConfiguredCapacityMetrics(resourceCalculator, - labelManager.getResourceByLabel(null, clusterResource), RMNodeLabelsManager.NO_LABEL, - this); + labelManager.getResourceByLabel(null, clusterResource), + RMNodeLabelsManager.NO_LABEL, this); // queue metrics are updated, more resource may be available // activate the pending applications if possible @@ -1890,8 +1977,10 @@ public void updateClusterResource(Resource clusterResource, usersManager.userLimitNeedsRecompute(); // Update application properties - for (FiCaSchedulerApp application : orderingPolicy.getSchedulableEntities()) { - computeUserLimitAndSetHeadroom(application, clusterResource, RMNodeLabelsManager.NO_LABEL, + for (FiCaSchedulerApp application : orderingPolicy + .getSchedulableEntities()) { + computeUserLimitAndSetHeadroom(application, clusterResource, + RMNodeLabelsManager.NO_LABEL, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, null); } } finally { @@ -1902,14 +1991,16 @@ public void updateClusterResource(Resource clusterResource, @Override public void incUsedResource(String nodeLabel, Resource resourceToInc, SchedulerApplicationAttempt application) { - usersManager.updateUserResourceUsage(application.getUser(), resourceToInc, nodeLabel, true); + usersManager.updateUserResourceUsage(application.getUser(), resourceToInc, + queueContext.getClusterResource(), nodeLabel, true); super.incUsedResource(nodeLabel, resourceToInc, application); } @Override public void decUsedResource(String nodeLabel, Resource resourceToDec, SchedulerApplicationAttempt application) { - usersManager.updateUserResourceUsage(application.getUser(), resourceToDec, nodeLabel, false); + usersManager.updateUserResourceUsage(application.getUser(), resourceToDec, + queueContext.getClusterResource(), nodeLabel, false); super.decUsedResource(nodeLabel, resourceToDec, application); } @@ -1920,7 +2011,8 @@ public void incAMUsedResource(String nodeLabel, Resource resourceToInc, return; } - user.getResourceUsage().incAMUsed(nodeLabel, resourceToInc); + user.getResourceUsage().incAMUsed(nodeLabel, + resourceToInc); // ResourceUsage has its own lock, no addition lock needs here. usageTracker.getQueueUsage().incAMUsed(nodeLabel, resourceToInc); } @@ -1932,7 +2024,8 @@ public void decAMUsedResource(String nodeLabel, Resource resourceToDec, return; } - user.getResourceUsage().decAMUsed(nodeLabel, resourceToDec); + user.getResourceUsage().decAMUsed(nodeLabel, + resourceToDec); // ResourceUsage has its own lock, no addition lock needs here. usageTracker.getQueueUsage().decAMUsed(nodeLabel, resourceToDec); } @@ -1949,9 +2042,11 @@ public void recoverContainer(Resource clusterResource, // Careful! Locking order is important! writeLock.lock(); try { - FiCaSchedulerNode node = csContext.getNode(rmContainer.getContainer().getNodeId()); - allocateResource(clusterResource, attempt, rmContainer.getContainer().getResource(), - node.getPartition(), rmContainer); + FiCaSchedulerNode node = queueContext.getNode( + rmContainer.getContainer().getNodeId()); + allocateResource(clusterResource, attempt, + rmContainer.getContainer().getResource(), node.getPartition(), + rmContainer); } finally { writeLock.unlock(); } @@ -1963,22 +2058,24 @@ public void recoverContainer(Resource clusterResource, * Obtain (read-only) collection of pending applications. */ public Collection getPendingApplications() { - return Collections.unmodifiableCollection(pendingOrderingPolicy.getSchedulableEntities()); + return Collections.unmodifiableCollection(pendingOrderingPolicy + .getSchedulableEntities()); } /** * Obtain (read-only) collection of active applications. */ public Collection getApplications() { - return Collections.unmodifiableCollection(orderingPolicy.getSchedulableEntities()); + return Collections.unmodifiableCollection(orderingPolicy + .getSchedulableEntities()); } /** * Obtain (read-only) collection of all applications. */ public Collection getAllApplications() { - Collection apps = - new HashSet(pendingOrderingPolicy.getSchedulableEntities()); + Collection apps = new HashSet( + pendingOrderingPolicy.getSchedulableEntities()); apps.addAll(orderingPolicy.getSchedulableEntities()); return Collections.unmodifiableCollection(apps); @@ -1992,6 +2089,7 @@ public Collection getAllApplications() { * Total pending for the queue = * sum(for each user(min((user's headroom), sum(user's pending requests)))) * NOTE: + * @param clusterResources clusterResource * @param partition node partition * @param deductReservedFromPending When a container is reserved in CS, @@ -2071,7 +2169,7 @@ public void attachContainer(Resource clusterResource, if (application != null && rmContainer != null && rmContainer.getExecutionType() == ExecutionType.GUARANTEED) { FiCaSchedulerNode node = - csContext.getNode(rmContainer.getContainer().getNodeId()); + queueContext.getNode(rmContainer.getContainer().getNodeId()); allocateResource(clusterResource, application, rmContainer.getContainer() .getResource(), node.getPartition(), rmContainer); LOG.info("movedContainer" + " container=" + rmContainer.getContainer() @@ -2089,9 +2187,9 @@ public void attachContainer(Resource clusterResource, public void detachContainer(Resource clusterResource, FiCaSchedulerApp application, RMContainer rmContainer) { if (application != null && rmContainer != null - && rmContainer.getExecutionType() == ExecutionType.GUARANTEED) { + && rmContainer.getExecutionType() == ExecutionType.GUARANTEED) { FiCaSchedulerNode node = - csContext.getNode(rmContainer.getContainer().getNodeId()); + queueContext.getNode(rmContainer.getContainer().getNodeId()); releaseResource(clusterResource, application, rmContainer.getContainer() .getResource(), node.getPartition(), rmContainer); LOG.info("movedContainer" + " container=" + rmContainer.getContainer() @@ -2110,7 +2208,7 @@ public void detachContainer(Resource clusterResource, * this will be used by preemption policy. */ public Map> - getIgnoreExclusivityRMContainers() { + getIgnoreExclusivityRMContainers() { Map> clonedMap = new HashMap<>(); readLock.lock(); @@ -2156,8 +2254,7 @@ public void setMaxAMResourcePerQueuePercent( this.maxAMResourcePerQueuePercent = maxAMResourcePerQueuePercent; } - public OrderingPolicy - getOrderingPolicy() { + public OrderingPolicy getOrderingPolicy() { return orderingPolicy; } @@ -2202,8 +2299,7 @@ public void updateApplicationPriority(SchedulerApplication app } } - public OrderingPolicy - getPendingAppsOrderingPolicy() { + public OrderingPolicy getPendingAppsOrderingPolicy() { return pendingOrderingPolicy; } @@ -2376,7 +2472,7 @@ List getCopyOfNonRunnableAppSchedulables() { @Override public boolean isEligibleForAutoDeletion() { return isDynamicQueue() && getNumApplications() == 0 - && csContext.getConfiguration(). + && queueContext.getConfiguration(). isAutoExpiredDeletionEnabled(this.getQueuePath()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java index 7d149761cb0dba..6d272184100bad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java @@ -44,9 +44,9 @@ public abstract class AbstractManagedParentQueue extends ParentQueue { protected AutoCreatedLeafQueueConfig leafQueueTemplate; protected AutoCreatedQueueManagementPolicy queueManagementPolicy = null; - public AbstractManagedParentQueue(CapacitySchedulerContext cs, + public AbstractManagedParentQueue(CapacitySchedulerQueueContext queueContext, String queueName, CSQueue parent, CSQueue old) throws IOException { - super(cs, queueName, parent, old); + super(queueContext, queueName, parent, old); } @Override @@ -55,7 +55,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) writeLock.lock(); try { // Set new configs - setupQueueConfigs(clusterResource, csContext.getConfiguration()); + setupQueueConfigs(clusterResource, queueContext.getConfiguration()); } finally { writeLock.unlock(); @@ -121,8 +121,7 @@ public CSQueue removeChildQueue(String childQueueName) CSQueue childQueue; writeLock.lock(); try { - childQueue = this.csContext.getCapacitySchedulerQueueManager().getQueue( - childQueueName); + childQueue = queueContext.getQueueManager().getQueue(childQueueName); if (childQueue != null) { removeChildQueue(childQueue); } else { @@ -176,14 +175,14 @@ protected CapacitySchedulerConfiguration initializeLeafQueueConfigs(String CapacitySchedulerConfiguration leafQueueConfigs = new CapacitySchedulerConfiguration(new Configuration(false), false); - Map rtProps = csContext + Map rtProps = queueContext .getConfiguration().getConfigurationProperties() .getPropertiesWithPrefix(YarnConfiguration.RESOURCE_TYPES + ".", true); for (Map.Entry entry : rtProps.entrySet()) { leafQueueConfigs.set(entry.getKey(), entry.getValue()); } - Map templateConfigs = csContext + Map templateConfigs = queueContext .getConfiguration().getConfigurationProperties() .getPropertiesWithPrefix(configPrefix, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java index 57050b193abd68..2d818cd1556a58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java @@ -39,16 +39,11 @@ public class AutoCreatedLeafQueue extends AbstractAutoCreatedLeafQueue { private static final Logger LOG = LoggerFactory .getLogger(AutoCreatedLeafQueue.class); - public AutoCreatedLeafQueue(CapacitySchedulerContext cs, String queueName, + public AutoCreatedLeafQueue(CapacitySchedulerQueueContext queueContext, String queueName, ManagedParentQueue parent) throws IOException { - // TODO once YARN-10907 is merged the duplicated collection of - // leafQueueConfigs won't be necessary - super(cs, parent.getLeafQueueConfigs(queueName), - queueName, - parent, null); - super.setupQueueConfigs(cs.getClusterResource(), parent.getLeafQueueConfigs(queueName)); - - LOG.debug("Initialized AutoCreatedLeafQueue: name={}, fullname={}", queueName, getQueuePath()); + super(queueContext, queueName, parent, null); + super.setupQueueConfigs(queueContext.getClusterResource(), parent.getLeafQueueConfigs(queueName)); + updateCapacitiesToZero(); } @@ -74,8 +69,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) } } - public void reinitializeFromTemplate(AutoCreatedLeafQueueConfig - leafQueueTemplate) throws SchedulerDynamicEditException, IOException { + public void reinitializeFromTemplate(AutoCreatedLeafQueueConfig leafQueueTemplate) { writeLock.lock(); try { @@ -105,7 +99,7 @@ public void mergeCapacities(QueueCapacities capacities) { .getAbsoluteMaximumCapacity(nodeLabel)); Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel, - csContext.getClusterResource()); + queueContext.getClusterResource()); getQueueResourceQuotas().setEffectiveMinResource(nodeLabel, Resources.multiply(resourceByLabel, queueCapacities.getAbsoluteCapacity(nodeLabel))); @@ -133,12 +127,12 @@ protected void setDynamicQueueProperties( String parentTemplate = String.format("%s.%s", getParent().getQueuePath(), CapacitySchedulerConfiguration .AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX); - Set parentNodeLabels = csContext - .getCapacitySchedulerQueueManager().getConfiguredNodeLabels() + Set parentNodeLabels = queueContext + .getQueueManager().getConfiguredNodeLabelsForAllQueues() .getLabelsByQueue(parentTemplate); if (parentNodeLabels != null && parentNodeLabels.size() > 1) { - csContext.getCapacitySchedulerQueueManager().getConfiguredNodeLabels() + queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues() .setLabelsByQueue(getQueuePath(), new HashSet<>(parentNodeLabels)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueManagementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueManagementPolicy.java index 388e9d6233bd45..bf99c3f30c3ec4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueManagementPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueManagementPolicy.java @@ -27,17 +27,15 @@ public interface AutoCreatedQueueManagementPolicy { /** * Initialize policy - * @param schedulerContext Capacity Scheduler context + * @param parentQueue parent queue */ - void init(CapacitySchedulerContext schedulerContext, ParentQueue - parentQueue) throws IOException; + void init(ParentQueue parentQueue) throws IOException; /** * Reinitialize policy state ( if required ) - * @param schedulerContext Capacity Scheduler context + * @param parentQueue parent queue */ - void reinitialize(CapacitySchedulerContext schedulerContext, - ParentQueue parentQueue) throws IOException; + void reinitialize(ParentQueue parentQueue) throws IOException; /** * Get initial template for the specified leaf queue diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java index 2cfd5a4310f82d..56874888870b86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java @@ -26,11 +26,13 @@ public class CSQueuePreemptionSettings { public CSQueuePreemptionSettings( CSQueue queue, - CapacitySchedulerContext csContext, - CapacitySchedulerConfiguration configuration) { - this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(queue, csContext, configuration); + CapacitySchedulerConfiguration configuration, + CapacitySchedulerConfiguration originalSchedulerConfiguration) { + this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(queue, configuration, + originalSchedulerConfiguration); this.intraQueuePreemptionDisabledInHierarchy = - isIntraQueueHierarchyPreemptionDisabled(queue, csContext, configuration); + isIntraQueueHierarchyPreemptionDisabled(queue, configuration, + originalSchedulerConfiguration); } /** @@ -40,14 +42,14 @@ public CSQueuePreemptionSettings( * NOTE: Cross-queue preemptability is inherited from a queue's parent. * * @param q queue to check preemption state - * @param csContext * @param configuration capacity scheduler config * @return true if queue has cross-queue preemption disabled, false otherwise */ private boolean isQueueHierarchyPreemptionDisabled(CSQueue q, - CapacitySchedulerContext csContext, CapacitySchedulerConfiguration configuration) { + CapacitySchedulerConfiguration configuration, + CapacitySchedulerConfiguration originalSchedulerConfiguration) { boolean systemWidePreemption = - csContext.getConfiguration() + originalSchedulerConfiguration .getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS); CSQueue parentQ = q.getParent(); @@ -79,14 +81,14 @@ private boolean isQueueHierarchyPreemptionDisabled(CSQueue q, * NOTE: Intra-queue preemptability is inherited from a queue's parent. * * @param q queue to check intra-queue preemption state - * @param csContext * @param configuration capacity scheduler config * @return true if queue has intra-queue preemption disabled, false otherwise */ private boolean isIntraQueueHierarchyPreemptionDisabled(CSQueue q, - CapacitySchedulerContext csContext, CapacitySchedulerConfiguration configuration) { + CapacitySchedulerConfiguration configuration, + CapacitySchedulerConfiguration originalSchedulerConfiguration) { boolean systemWideIntraQueuePreemption = - csContext.getConfiguration().getBoolean( + originalSchedulerConfiguration.getBoolean( CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ENABLED, CapacitySchedulerConfiguration .DEFAULT_INTRAQUEUE_PREEMPTION_ENABLED); @@ -109,7 +111,7 @@ private boolean isIntraQueueHierarchyPreemptionDisabled(CSQueue q, parentQ.getIntraQueuePreemptionDisabledInHierarchy()); } - public boolean getIntraQueuePreemptionDisabled() { + public boolean isIntraQueuePreemptionDisabled() { return intraQueuePreemptionDisabledInHierarchy || preemptionDisabled; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index befb82a70e22e3..abd40a8062b530 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -183,6 +183,8 @@ public class CapacityScheduler extends private CapacitySchedulerQueueManager queueManager; + private CapacitySchedulerQueueContext queueContext; + private WorkflowPriorityMappingsManager workflowPriorityMappingsMgr; // timeout to join when we stop this service @@ -267,6 +269,11 @@ public CapacitySchedulerConfiguration getConfiguration() { return conf; } + @Override + public CapacitySchedulerQueueContext getQueueContext() { + return queueContext; + } + @Override public RMContainerTokenSecretManager getContainerTokenSecretManager() { return this.rmContext.getContainerTokenSecretManager(); @@ -319,6 +326,7 @@ void initScheduler(Configuration configuration) throws this.workflowPriorityMappingsMgr = new WorkflowPriorityMappingsManager(); this.activitiesManager = new ActivitiesManager(rmContext); activitiesManager.init(conf); + this.queueContext = new CapacitySchedulerQueueContext(this); initializeQueues(this.conf); this.isLazyPreemptionEnabled = conf.getLazyPreemptionEnabled(); this.assignMultipleEnabled = this.conf.getAssignMultipleEnabled(); @@ -844,6 +852,7 @@ private void initializeQueues(CapacitySchedulerConfiguration conf) @Lock(CapacityScheduler.class) private void reinitializeQueues(CapacitySchedulerConfiguration newConf) throws IOException { + queueContext.reinitialize(); this.queueManager.reinitializeQueues(newConf); updatePlacementRules(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index e88f83a44cbc2b..2716ddebbdc6db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -1658,6 +1658,12 @@ public Map> getConfiguredNodeLabelsByQueue() { return labelsByQueue; } + public Priority getClusterLevelApplicationMaxPriority() { + return Priority.newInstance(getInt( + YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, + YarnConfiguration.DEFAULT_CLUSTER_LEVEL_APPLICATION_PRIORITY)); + } + public Integer getDefaultApplicationPriorityConfPerQueue(String queue) { Integer defaultPriority = getInt(getQueuePrefix(queue) + DEFAULT_APPLICATION_PRIORITY, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java index ae74989a7265fb..1d0600f66807eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java @@ -41,6 +41,8 @@ */ public interface CapacitySchedulerContext { CapacitySchedulerConfiguration getConfiguration(); + + CapacitySchedulerQueueContext getQueueContext(); Resource getMinimumResourceCapability(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java new file mode 100644 index 00000000000000..e9ec3a0e490649 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerHealth; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; + +/** + * Class to store common queue related information, like instances + * to necessary manager classes or the global CapacityScheduler + * configuration. + */ +public class CapacitySchedulerQueueContext { + + // Manager classes + private final CapacitySchedulerContext csContext; + private final CapacitySchedulerQueueManager queueManager; + private final RMNodeLabelsManager labelManager; + private final PreemptionManager preemptionManager; + private final ActivitiesManager activitiesManager; + private final ResourceCalculator resourceCalculator; + + // CapacityScheduler configuration + private CapacitySchedulerConfiguration configuration; + + private Resource minimumAllocation; + + public CapacitySchedulerQueueContext(CapacitySchedulerContext csContext) { + this.csContext = csContext; + this.queueManager = csContext.getCapacitySchedulerQueueManager(); + this.labelManager = csContext.getRMContext().getNodeLabelManager(); + this.preemptionManager = csContext.getPreemptionManager(); + this.activitiesManager = csContext.getActivitiesManager(); + this.resourceCalculator = csContext.getResourceCalculator(); + + this.configuration = new CapacitySchedulerConfiguration(csContext.getConfiguration()); + this.minimumAllocation = csContext.getMinimumResourceCapability(); + } + + public void reinitialize() { + // When csConfProvider.loadConfiguration is called, the useLocalConfigurationProvider is + // correctly set to load the config entries from the capacity-scheduler.xml. + // For this reason there is no need to reload from it again. + this.configuration = new CapacitySchedulerConfiguration(csContext.getConfiguration(), false); + this.minimumAllocation = csContext.getMinimumResourceCapability(); + } + + public CapacitySchedulerQueueManager getQueueManager() { + return queueManager; + } + + public RMNodeLabelsManager getLabelManager() { + return labelManager; + } + + public PreemptionManager getPreemptionManager() { + return preemptionManager; + } + + public ActivitiesManager getActivitiesManager() { + return activitiesManager; + } + + public ResourceCalculator getResourceCalculator() { + return resourceCalculator; + } + + public CapacitySchedulerConfiguration getConfiguration() { + return configuration; + } + + public Resource getMinimumAllocation() { + return minimumAllocation; + } + + public Resource getClusterResource() { + return csContext.getClusterResource(); + } + + public ResourceUsage getClusterResourceUsage() { + return queueManager.getRootQueue().getQueueResourceUsage(); + } + + public SchedulerHealth getSchedulerHealth() { + return csContext.getSchedulerHealth(); + } + + public long getLastNodeUpdateTime() { + return csContext.getLastNodeUpdateTime(); + } + + public FiCaSchedulerNode getNode(NodeId nodeId) { + return csContext.getNode(nodeId); + } + + public FiCaSchedulerApp getApplicationAttempt( + ApplicationAttemptId applicationAttemptId) { + return csContext.getApplicationAttempt(applicationAttemptId); + } + + // TODO this is used in GuaranteedOrZeroCapacityOverTimePolicy, refactor the comparator there + public RMApp getRMApp(ApplicationId applicationId) { + return csContext.getRMContext().getRMApps().get(applicationId); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index 407383d3bc0fcf..c1669d0c763b23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -106,6 +106,11 @@ public CSQueue getRootQueue() { return this.root; } + @VisibleForTesting + protected void setRootQueue(CSQueue rootQueue) { + this.root = rootQueue; + } + @Override public Map getQueues() { return queues.getFullNameQueues(); @@ -167,7 +172,7 @@ public void setCapacitySchedulerContext( public void initializeQueues(CapacitySchedulerConfiguration conf) throws IOException { configuredNodeLabels = new ConfiguredNodeLabels(conf); - root = parseQueue(this.csContext, conf, null, + root = parseQueue(this.csContext.getQueueContext(), conf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, NOOP); setQueueAcls(authorizer, appPriorityACLManager, queues); labelManager.reinitializeQueueLabels(getQueueToLabels()); @@ -183,7 +188,7 @@ public void reinitializeQueues(CapacitySchedulerConfiguration newConf) // Parse new queues CSQueueStore newQueues = new CSQueueStore(); configuredNodeLabels = new ConfiguredNodeLabels(newConf); - CSQueue newRoot = parseQueue(this.csContext, newConf, null, + CSQueue newRoot = parseQueue(this.csContext.getQueueContext(), newConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, NOOP); // When failing over, if using configuration store, don't validate queue @@ -215,7 +220,7 @@ public void reinitializeQueues(CapacitySchedulerConfiguration newConf) /** * Parse the queue from the configuration. - * @param csContext the CapacitySchedulerContext + * @param queueContext the CapacitySchedulerQueueContext * @param conf the CapacitySchedulerConfiguration * @param parent the parent queue * @param queueName the queue name @@ -226,7 +231,7 @@ public void reinitializeQueues(CapacitySchedulerConfiguration newConf) * @throws IOException */ static CSQueue parseQueue( - CapacitySchedulerContext csContext, + CapacitySchedulerQueueContext queueContext, CapacitySchedulerConfiguration conf, CSQueue parent, String queueName, CSQueueStore newQueues, @@ -265,7 +270,7 @@ static CSQueue parseQueue( // Check if the queue will be dynamically managed by the Reservation // system if (isReservableQueue) { - queue = new PlanQueue(csContext, queueName, parent, + queue = new PlanQueue(queueContext, queueName, parent, oldQueues.get(fullQueueName)); //initializing the "internal" default queue, for SLS compatibility @@ -273,7 +278,7 @@ static CSQueue parseQueue( queueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX; List childQueues = new ArrayList<>(); - ReservationQueue resQueue = new ReservationQueue(csContext, + ReservationQueue resQueue = new ReservationQueue(queueContext, defReservationId, (PlanQueue) queue); try { resQueue.setEntitlement(new QueueEntitlement(1.0f, 1.0f)); @@ -285,11 +290,11 @@ static CSQueue parseQueue( newQueues.add(resQueue); } else if (isAutoCreateEnabled) { - queue = new ManagedParentQueue(csContext, queueName, parent, + queue = new ManagedParentQueue(queueContext, queueName, parent, oldQueues.get(fullQueueName)); } else{ - queue = new LeafQueue(csContext, queueName, parent, + queue = new LeafQueue(queueContext, queueName, parent, oldQueues.get(fullQueueName)); // Used only for unit tests queue = hook.hook(queue); @@ -302,10 +307,10 @@ static CSQueue parseQueue( ParentQueue parentQueue; if (isAutoCreateEnabled) { - parentQueue = new ManagedParentQueue(csContext, queueName, parent, + parentQueue = new ManagedParentQueue(queueContext, queueName, parent, oldQueues.get(fullQueueName)); } else{ - parentQueue = new ParentQueue(csContext, queueName, parent, + parentQueue = new ParentQueue(queueContext, queueName, parent, oldQueues.get(fullQueueName)); } @@ -314,7 +319,7 @@ static CSQueue parseQueue( List childQueues = new ArrayList<>(); for (String childQueueName : childQueueNames) { - CSQueue childQueue = parseQueue(csContext, conf, queue, childQueueName, + CSQueue childQueue = parseQueue(queueContext, conf, queue, childQueueName, newQueues, oldQueues, hook); childQueues.add(childQueue); } @@ -633,7 +638,7 @@ public List determineMissingParents( * for all queues. * @return configured node labels */ - public ConfiguredNodeLabels getConfiguredNodeLabels() { + public ConfiguredNodeLabels getConfiguredNodeLabelsForAllQueues() { return configuredNodeLabels; } @@ -676,7 +681,7 @@ private AbstractLeafQueue createLegacyAutoQueue(QueuePath queue) (ManagedParentQueue) parentQueue; AutoCreatedLeafQueue autoCreatedLeafQueue = new AutoCreatedLeafQueue( - csContext, queue.getLeafName(), autoCreateEnabledParentQueue); + csContext.getQueueContext(), queue.getLeafName(), autoCreateEnabledParentQueue); addLegacyDynamicQueue(autoCreatedLeafQueue); return autoCreatedLeafQueue; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index b9fa932f14141d..ee53c14f8b0b00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -31,26 +31,16 @@ public class LeafQueue extends AbstractLeafQueue { private static final Logger LOG = LoggerFactory.getLogger(LeafQueue.class); - @SuppressWarnings({ "unchecked", "rawtypes" }) - public LeafQueue(CapacitySchedulerContext cs, + public LeafQueue(CapacitySchedulerQueueContext queueContext, String queueName, CSQueue parent, CSQueue old) throws IOException { - this(cs, cs.getConfiguration(), queueName, parent, old, false); + this(queueContext, queueName, parent, old, false); } - public LeafQueue(CapacitySchedulerContext cs, - CapacitySchedulerConfiguration configuration, - String queueName, CSQueue parent, CSQueue old) throws IOException { - this(cs, configuration, queueName, parent, old, false); - } - - public LeafQueue(CapacitySchedulerContext cs, - CapacitySchedulerConfiguration configuration, + public LeafQueue(CapacitySchedulerQueueContext queueContext, String queueName, CSQueue parent, CSQueue old, boolean isDynamic) throws IOException { - super(cs, configuration, queueName, parent, old, isDynamic); - - setupQueueConfigs(cs.getClusterResource(), configuration); + super(queueContext, queueName, parent, old, isDynamic); - LOG.debug("LeafQueue: name={}, fullname={}", queueName, getQueuePath()); + setupQueueConfigs(queueContext.getClusterResource(), queueContext.getConfiguration()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index ddfb24bf6fce69..0aab2e412f3966 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -53,23 +53,18 @@ public class ManagedParentQueue extends AbstractManagedParentQueue { private static final Logger LOG = LoggerFactory.getLogger( ManagedParentQueue.class); - public ManagedParentQueue(final CapacitySchedulerContext cs, + public ManagedParentQueue(final CapacitySchedulerQueueContext queueContext, final String queueName, final CSQueue parent, final CSQueue old) throws IOException { - super(cs, queueName, parent, old); + super(queueContext, queueName, parent, old); shouldFailAutoCreationWhenGuaranteedCapacityExceeded = - csContext.getConfiguration() + queueContext.getConfiguration() .getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded( getQueuePath()); leafQueueTemplate = initializeLeafQueueConfigs().build(); - LOG.info( - "Created Managed Parent Queue: [{}] with capacity: [{}]" - + " with max capacity: [{}]", - queueName, super.getCapacity(), super.getMaximumCapacity()); - initializeQueueManagementPolicy(); } @@ -82,7 +77,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) validate(newlyParsedQueue); shouldFailAutoCreationWhenGuaranteedCapacityExceeded = - csContext.getConfiguration() + queueContext.getConfiguration() .getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded( getQueuePath()); @@ -133,23 +128,23 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) private void initializeQueueManagementPolicy() throws IOException { queueManagementPolicy = - csContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass( + queueContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass( getQueuePath()); - queueManagementPolicy.init(csContext, this); + queueManagementPolicy.init(this); } private void reinitializeQueueManagementPolicy() throws IOException { AutoCreatedQueueManagementPolicy managementPolicy = - csContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass( + queueContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass( getQueuePath()); if (!(managementPolicy.getClass().equals( this.queueManagementPolicy.getClass()))) { queueManagementPolicy = managementPolicy; - queueManagementPolicy.init(csContext, this); + queueManagementPolicy.init(this); } else{ - queueManagementPolicy.reinitialize(csContext, this); + queueManagementPolicy.reinitialize(this); } } @@ -158,21 +153,25 @@ protected AutoCreatedLeafQueueConfig.Builder initializeLeafQueueConfigs() throws AutoCreatedLeafQueueConfig.Builder builder = new AutoCreatedLeafQueueConfig.Builder(); + CapacitySchedulerConfiguration configuration = + queueContext.getConfiguration(); + + // TODO load configs into CapacitySchedulerConfiguration instead of duplicating them String leafQueueTemplateConfPrefix = getLeafQueueConfigPrefix( - csContext.getConfiguration()); - //Load template configuration - CapacitySchedulerConfiguration conf = + configuration); + //Load template configuration into CapacitySchedulerConfiguration + CapacitySchedulerConfiguration autoCreatedTemplateConfig = super.initializeLeafQueueConfigs(leafQueueTemplateConfPrefix); - builder.configuration(conf); - QueuePath templateQueuePath = csContext.getConfiguration() + builder.configuration(autoCreatedTemplateConfig); + QueuePath templateQueuePath = configuration .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePath()); - Set templateConfiguredNodeLabels = csContext - .getCapacitySchedulerQueueManager().getConfiguredNodeLabels() + Set templateConfiguredNodeLabels = queueContext + .getQueueManager().getConfiguredNodeLabelsForAllQueues() .getLabelsByQueue(templateQueuePath.getFullPath()); for (String nodeLabel : templateConfiguredNodeLabels) { - Resource templateMinResource = conf.getMinimumResourceRequirement( - nodeLabel, csContext.getConfiguration() + Resource templateMinResource = autoCreatedTemplateConfig.getMinimumResourceRequirement( + nodeLabel, configuration .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()), resourceTypes); @@ -187,7 +186,7 @@ protected AutoCreatedLeafQueueConfig.Builder initializeLeafQueueConfigs() throws QueueCapacities queueCapacities = new QueueCapacities(false); CSQueueUtils.loadCapacitiesByLabelsFromConf(templateQueuePath, queueCapacities, - csContext.getConfiguration(), + configuration, templateConfiguredNodeLabels); @@ -205,35 +204,38 @@ protected AutoCreatedLeafQueueConfig.Builder initializeLeafQueueConfigs() throws } private void updateQueueCapacities(QueueCapacities queueCapacities) { + CapacitySchedulerConfiguration configuration = + queueContext.getConfiguration(); + for (String label : queueCapacities.getExistingNodeLabels()) { queueCapacities.setCapacity(label, - this.csContext.getResourceCalculator().divide( - this.csContext.getClusterResource(), - this.csContext.getConfiguration().getMinimumResourceRequirement( + resourceCalculator.divide( + queueContext.getClusterResource(), + configuration.getMinimumResourceRequirement( label, - this.csContext.getConfiguration() + configuration .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()), resourceTypes), getQueueResourceQuotas().getConfiguredMinResource(label))); - Resource childMaxResource = this.csContext.getConfiguration() + Resource childMaxResource = configuration .getMaximumResourceRequirement(label, - this.csContext.getConfiguration() + configuration .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()), resourceTypes); Resource parentMaxRes = getQueueResourceQuotas() .getConfiguredMaxResource(label); Resource effMaxResource = Resources.min( - this.csContext.getResourceCalculator(), - this.csContext.getClusterResource(), + resourceCalculator, + queueContext.getClusterResource(), childMaxResource.equals(Resources.none()) ? parentMaxRes : childMaxResource, parentMaxRes); queueCapacities.setMaximumCapacity( - label, this.csContext.getResourceCalculator().divide( - this.csContext.getClusterResource(), + label, resourceCalculator.divide( + queueContext.getClusterResource(), effMaxResource, getQueueResourceQuotas().getConfiguredMaxResource(label))); @@ -268,7 +270,7 @@ public void addChildQueue(CSQueue childQueue) "Expected child queue to be an instance of AutoCreatedLeafQueue"); } - CapacitySchedulerConfiguration conf = csContext.getConfiguration(); + CapacitySchedulerConfiguration conf = queueContext.getConfiguration(); ManagedParentQueue parentQueue = (ManagedParentQueue) childQueue.getParent(); @@ -322,8 +324,8 @@ public void addChildQueue(CSQueue childQueue) // Do one update cluster resource call to make sure all absolute resources // effective resources are updated. - updateClusterResource(this.csContext.getClusterResource(), - new ResourceLimits(this.csContext.getClusterResource())); + updateClusterResource(queueContext.getClusterResource(), + new ResourceLimits(queueContext.getClusterResource())); } finally { writeLock.unlock(); } @@ -427,12 +429,11 @@ public void validateQueueManagementChanges( + " Ignoring update " + queueManagementChanges); } - switch (queueManagementChange.getQueueAction()){ - case UPDATE_QUEUE: + if (queueManagementChange.getQueueAction() == + QueueManagementChange.QueueAction.UPDATE_QUEUE) { AutoCreatedLeafQueueConfig template = queueManagementChange.getUpdatedQueueTemplate(); ((AutoCreatedLeafQueue) childQueue).validateConfigurations(template); - break; } } @@ -442,14 +443,13 @@ private void applyQueueManagementChanges( List queueManagementChanges) throws SchedulerDynamicEditException, IOException { for (QueueManagementChange queueManagementChange : queueManagementChanges) { - switch (queueManagementChange.getQueueAction()){ - case UPDATE_QUEUE: + if (queueManagementChange.getQueueAction() == + QueueManagementChange.QueueAction.UPDATE_QUEUE) { AutoCreatedLeafQueue childQueueToBeUpdated = (AutoCreatedLeafQueue) queueManagementChange.getQueue(); //acquires write lock on leaf queue childQueueToBeUpdated.reinitializeFromTemplate( queueManagementChange.getUpdatedQueueTemplate()); - break; } } } @@ -465,7 +465,7 @@ public CapacitySchedulerConfiguration getLeafQueueConfigs( CapacitySchedulerConfiguration leafQueueConfigTemplate = new CapacitySchedulerConfiguration(new Configuration(false), false); for (final Iterator> iterator = - templateConfig.iterator(); iterator.hasNext(); ) { + templateConfig.iterator(); iterator.hasNext();) { Map.Entry confKeyValuePair = iterator.next(); final String name = confKeyValuePair.getKey().replaceFirst( CapacitySchedulerConfiguration diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 43391897cdfad9..b2ff8995ff3cfd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -87,7 +87,6 @@ public class ParentQueue extends AbstractCSQueue { protected final List childQueues; private final boolean rootQueue; private volatile int numApplications; - private final CapacitySchedulerContext scheduler; private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); @@ -108,27 +107,20 @@ public class ParentQueue extends AbstractCSQueue { // after every time recalculation private volatile Map effectiveMinRatioPerResource; - public ParentQueue(CapacitySchedulerContext cs, + public ParentQueue(CapacitySchedulerQueueContext queueContext, String queueName, CSQueue parent, CSQueue old) throws IOException { - this(cs, cs.getConfiguration(), queueName, parent, old, false); - } - private ParentQueue(CapacitySchedulerContext cs, - CapacitySchedulerConfiguration csConf, String queueName, - CSQueue parent, - CSQueue old) throws IOException { - this(cs, csConf, queueName, parent, old, false); + this(queueContext, queueName, parent, old, false); } - private ParentQueue(CapacitySchedulerContext cs, - CapacitySchedulerConfiguration csConf, String queueName, CSQueue parent, - CSQueue old, boolean isDynamic) + private ParentQueue(CapacitySchedulerQueueContext queueContext, + String queueName, CSQueue parent, CSQueue old, boolean isDynamic) throws IOException { - super(cs, queueName, parent, old); + super(queueContext, queueName, parent, old); setDynamicQueue(isDynamic); - this.scheduler = cs; this.rootQueue = (parent == null); - float rawCapacity = csConf.getNonLabeledQueueCapacity(this.queuePath); + float rawCapacity = queueContext.getConfiguration() + .getNonLabeledQueueCapacity(this.queuePath); if (rootQueue && (rawCapacity != CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE)) { @@ -139,13 +131,10 @@ private ParentQueue(CapacitySchedulerContext cs, this.childQueues = new ArrayList<>(); this.allowZeroCapacitySum = - cs.getConfiguration().getAllowZeroCapacitySum(getQueuePath()); - - setupQueueConfigs(cs.getClusterResource(), csConf); + queueContext.getConfiguration() + .getAllowZeroCapacitySum(getQueuePath()); - LOG.info("Initialized parent-queue " + queueName + - " name=" + queueName + - ", fullname=" + getQueuePath()); + setupQueueConfigs(queueContext.getClusterResource(), queueContext.getConfiguration()); } // returns what is configured queue ordering policy @@ -156,41 +145,42 @@ private String getQueueOrderingPolicyConfigName() { } protected void setupQueueConfigs(Resource clusterResource, - CapacitySchedulerConfiguration csConf) + CapacitySchedulerConfiguration configuration) throws IOException { writeLock.lock(); try { autoCreatedQueueTemplate = new AutoCreatedQueueTemplate( - csConf, this.queuePath); - super.setupQueueConfigs(clusterResource, csConf); + configuration, this.queuePath); + super.setupQueueConfigs(clusterResource, configuration); StringBuilder aclsString = new StringBuilder(); - for (Map.Entry e : acls.entrySet()) { - aclsString.append(e.getKey() + ":" + e.getValue().getAclString()); + for (Map.Entry e : getACLs().entrySet()) { + aclsString.append(e.getKey()).append(":") + .append(e.getValue().getAclString()); } StringBuilder labelStrBuilder = new StringBuilder(); - if (queueNodeLabelsSettings.getAccessibleNodeLabels() != null) { - for (String nodeLabel : queueNodeLabelsSettings.getAccessibleNodeLabels()) { + if (getAccessibleNodeLabels() != null) { + for (String nodeLabel : getAccessibleNodeLabels()) { labelStrBuilder.append(nodeLabel).append(","); } } // Initialize queue ordering policy - queueOrderingPolicy = csConf.getQueueOrderingPolicy( + queueOrderingPolicy = configuration.getQueueOrderingPolicy( getQueuePath(), parent == null ? null : ((ParentQueue) parent).getQueueOrderingPolicyConfigName()); queueOrderingPolicy.setQueues(childQueues); LOG.info(getQueueName() + ", " + getCapacityOrWeightString() - + ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() - + ", maxCapacity=" + this.queueCapacities.getMaximumCapacity() - + ", absoluteMaxCapacity=" + this.queueCapacities - .getAbsoluteMaximumCapacity() + ", state=" + getState() + ", acls=" - + aclsString + ", labels=" + labelStrBuilder.toString() + "\n" - + ", reservationsContinueLooking=" + reservationsContinueLooking + + ", absoluteCapacity=" + getAbsoluteCapacity() + + ", maxCapacity=" + getMaximumCapacity() + + ", absoluteMaxCapacity=" + getAbsoluteMaximumCapacity() + + ", state=" + getState() + ", acls=" + + aclsString + ", labels=" + labelStrBuilder + "\n" + + ", reservationsContinueLooking=" + isReservationsContinueLooking() + ", orderingPolicy=" + getQueueOrderingPolicyConfigName() - + ", priority=" + priority + + ", priority=" + getPriority() + ", allowZeroCapacitySum=" + allowZeroCapacitySum); } finally { writeLock.unlock(); @@ -325,7 +315,7 @@ void setChildQueues(Collection childQueues) throws IOException { .getConfiguredMinResource(nodeLabel)); } Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel, - scheduler.getClusterResource()); + queueContext.getClusterResource()); Resource parentMinResource = usageTracker.getQueueResourceQuotas().getConfiguredMinResource(nodeLabel); if (!parentMinResource.equals(Resources.none()) && Resources.lessThan( @@ -488,11 +478,10 @@ private CSQueue createNewQueue(String childQueuePath, boolean isLeaf) childQueuePath.lastIndexOf(".") + 1); if (isLeaf) { - childQueue = new LeafQueue(csContext, csContext.getConfiguration(), + childQueue = new LeafQueue(queueContext, queueShortName, this, null, true); } else{ - childQueue = new ParentQueue(csContext, csContext.getConfiguration(), - queueShortName, this, null, true); + childQueue = new ParentQueue(queueContext, queueShortName, this, null, true); } childQueue.setDynamicQueue(true); // It should be sufficient now, we don't need to set more, because weights @@ -523,7 +512,7 @@ private CSQueue addDynamicChildQueue(String childQueuePath, boolean isLeaf) // should not happen, since it will be handled before calling this method) // , but we will move on. CSQueue queue = - csContext.getCapacitySchedulerQueueManager().getQueueByFullName( + queueContext.getQueueManager().getQueueByFullName( childQueuePath); if (queue != null) { LOG.warn( @@ -533,7 +522,7 @@ private CSQueue addDynamicChildQueue(String childQueuePath, boolean isLeaf) } // Check if the max queue limit is exceeded. - int maxQueues = csContext.getConfiguration(). + int maxQueues = queueContext.getConfiguration(). getAutoCreatedQueuesV2MaxChildQueuesLimit(getQueuePath()); if (childQueues.size() >= maxQueues) { throw new SchedulerDynamicEditException( @@ -564,8 +553,8 @@ private CSQueue addDynamicChildQueue(String childQueuePath, boolean isLeaf) // Call updateClusterResource. // Which will deal with all effectiveMin/MaxResource // Calculation - this.updateClusterResource(csContext.getClusterResource(), - new ResourceLimits(this.csContext.getClusterResource())); + this.updateClusterResource(queueContext.getClusterResource(), + new ResourceLimits(queueContext.getClusterResource())); return newQueue; } finally { @@ -596,14 +585,14 @@ public void removeChildQueue(CSQueue queue) // Now we can do remove and update this.childQueues.remove(queue); - this.scheduler.getCapacitySchedulerQueueManager() + queueContext.getQueueManager() .removeQueue(queue.getQueuePath()); // Call updateClusterResource, // which will deal with all effectiveMin/MaxResource // Calculation - this.updateClusterResource(csContext.getClusterResource(), - new ResourceLimits(this.csContext.getClusterResource())); + this.updateClusterResource(queueContext.getClusterResource(), + new ResourceLimits(queueContext.getClusterResource())); } finally { writeLock.unlock(); @@ -617,7 +606,7 @@ public void removeChildQueue(CSQueue queue) * false otherwise */ public boolean isEligibleForAutoQueueCreation() { - return isDynamicQueue() || csContext.getConfiguration(). + return isDynamicQueue() || queueContext.getConfiguration(). isAutoQueueCreationV2Enabled(getQueuePath()); } @@ -644,7 +633,7 @@ public void reinitialize(CSQueue newlyParsedQueue, ParentQueue newlyParsedParentQueue = (ParentQueue) newlyParsedQueue; // Set new configs - setupQueueConfigs(clusterResource, csContext.getConfiguration()); + setupQueueConfigs(clusterResource, queueContext.getConfiguration()); // Re-configure existing child queues and add new ones // The CS has already checked to ensure all existing child queues are present! @@ -685,7 +674,7 @@ public void reinitialize(CSQueue newlyParsedQueue, currentChildQueues.put(newChildQueueName, newChildQueue); // inform CapacitySchedulerQueueManager CapacitySchedulerQueueManager queueManager = - this.csContext.getCapacitySchedulerQueueManager(); + queueContext.getQueueManager(); queueManager.addQueue(newChildQueueName, newChildQueue); continue; } @@ -1399,7 +1388,7 @@ public void recoverContainer(Resource clusterResource, // Careful! Locking order is important! writeLock.lock(); try { - FiCaSchedulerNode node = scheduler.getNode( + FiCaSchedulerNode node = queueContext.getNode( rmContainer.getContainer().getNodeId()); allocateResource(clusterResource, rmContainer.getContainer().getResource(), node.getPartition()); @@ -1437,7 +1426,7 @@ public void attachContainer(Resource clusterResource, FiCaSchedulerApp application, RMContainer rmContainer) { if (application != null) { FiCaSchedulerNode node = - scheduler.getNode(rmContainer.getContainer().getNodeId()); + queueContext.getNode(rmContainer.getContainer().getNodeId()); allocateResource(clusterResource, rmContainer.getContainer() .getResource(), node.getPartition()); LOG.info("movedContainer" + " queueMoveIn=" + getQueuePath() @@ -1456,7 +1445,7 @@ public void detachContainer(Resource clusterResource, FiCaSchedulerApp application, RMContainer rmContainer) { if (application != null) { FiCaSchedulerNode node = - scheduler.getNode(rmContainer.getContainer().getNodeId()); + queueContext.getNode(rmContainer.getContainer().getNodeId()); super.releaseResource(clusterResource, rmContainer.getContainer().getResource(), node.getPartition()); @@ -1543,9 +1532,9 @@ private void killContainersToEnforceMaxQueueCapacity(String partition, while (Resources.greaterThan(resourceCalculator, partitionResource, usageTracker.getQueueUsage().getUsed(partition), maxResource)) { RMContainer toKillContainer = killableContainerIter.next(); - FiCaSchedulerApp attempt = csContext.getApplicationAttempt( + FiCaSchedulerApp attempt = queueContext.getApplicationAttempt( toKillContainer.getContainerId().getApplicationAttemptId()); - FiCaSchedulerNode node = csContext.getNode( + FiCaSchedulerNode node = queueContext.getNode( toKillContainer.getAllocatedNode()); if (null != attempt && null != node) { AbstractLeafQueue lq = attempt.getCSLeafQueue(); @@ -1656,7 +1645,7 @@ Map getEffectiveMinRatioPerResource() { @Override public boolean isEligibleForAutoDeletion() { return isDynamicQueue() && getChildQueues().size() == 0 && - csContext.getConfiguration(). + queueContext.getConfiguration(). isAutoExpiredDeletionEnabled(this.getQueuePath()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java index 3cddeac6a677d0..2b182e532f4a92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java @@ -40,17 +40,15 @@ public class PlanQueue extends AbstractManagedParentQueue { private int maxAppsPerUserForReservation; private float userLimit; private float userLimitFactor; - protected CapacitySchedulerContext schedulerContext; private boolean showReservationsAsQueues; - public PlanQueue(CapacitySchedulerContext cs, String queueName, + public PlanQueue(CapacitySchedulerQueueContext queueContext, String queueName, CSQueue parent, CSQueue old) throws IOException { - super(cs, queueName, parent, old); + super(queueContext, queueName, parent, old); updateAbsoluteCapacities(); - this.schedulerContext = cs; // Set the reservation queue attributes for the Plan - CapacitySchedulerConfiguration conf = cs.getConfiguration(); + CapacitySchedulerConfiguration conf = queueContext.getConfiguration(); String queuePath = super.getQueuePath(); int maxAppsForReservation = conf.getMaximumApplicationsPerQueue(queuePath); showReservationsAsQueues = conf.getShowReservationAsQueues(queuePath); @@ -106,7 +104,7 @@ public void reinitialize(CSQueue newlyParsedQueue, } // Set new configs - setupQueueConfigs(clusterResource, csContext.getConfiguration()); + setupQueueConfigs(clusterResource, queueContext.getConfiguration()); updateQuotas(newlyParsedParentQueue.userLimit, newlyParsedParentQueue.userLimitFactor, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java index 5a19a22635d9e4..730b797104fe3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java @@ -32,12 +32,13 @@ public class QueueAllocationSettings { private final Resource minimumAllocation; private Resource maximumAllocation; - public QueueAllocationSettings(CapacitySchedulerContext csContext) { - this.minimumAllocation = csContext.getMinimumResourceCapability(); + public QueueAllocationSettings(Resource minimumAllocation) { + this.minimumAllocation = minimumAllocation; } - void setupMaximumAllocation(CapacitySchedulerConfiguration csConf, String queuePath, - CSQueue parent, CapacitySchedulerContext csContext) { + void setupMaximumAllocation(CapacitySchedulerConfiguration configuration, + CapacitySchedulerConfiguration originalSchedulerConfiguration, String queuePath, + CSQueue parent) { /* YARN-10869: When using AutoCreatedLeafQueues, the passed configuration * object is a cloned one containing only the template configs * (see ManagedParentQueue#getLeafQueueConfigs). To ensure that the actual @@ -45,8 +46,8 @@ void setupMaximumAllocation(CapacitySchedulerConfiguration csConf, String queueP * be used. */ Resource clusterMax = ResourceUtils - .fetchMaximumAllocationFromConfig(csContext.getConfiguration()); - Resource queueMax = csConf.getQueueMaximumAllocation(queuePath); + .fetchMaximumAllocationFromConfig(originalSchedulerConfiguration); + Resource queueMax = configuration.getQueueMaximumAllocation(queuePath); maximumAllocation = Resources.clone( parent == null ? clusterMax : parent.getMaximumAllocation()); @@ -59,8 +60,8 @@ void setupMaximumAllocation(CapacitySchedulerConfiguration csConf, String queueP if (queueMax == Resources.none()) { // Handle backward compatibility - long queueMemory = csConf.getQueueMaximumAllocationMb(queuePath); - int queueVcores = csConf.getQueueMaximumAllocationVcores(queuePath); + long queueMemory = configuration.getQueueMaximumAllocationMb(queuePath); + int queueVcores = configuration.getQueueMaximumAllocationVcores(queuePath); if (queueMemory != UNDEFINED) { maximumAllocation.setMemorySize(queueMemory); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java index 827259f1ae97a1..8d64e17a5e7864 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java @@ -31,7 +31,6 @@ public class QueueNodeLabelsSettings { private final CSQueue parent; private final String queuePath; - private final CapacitySchedulerContext csContext; private Set accessibleLabels; private Set configuredNodeLabels; private String defaultLabelExpression; @@ -39,18 +38,18 @@ public class QueueNodeLabelsSettings { public QueueNodeLabelsSettings(CapacitySchedulerConfiguration configuration, CSQueue parent, String queuePath, - CapacitySchedulerContext csContext) throws IOException { + ConfiguredNodeLabels configuredNodeLabels) throws IOException { this.parent = parent; this.queuePath = queuePath; - this.csContext = csContext; - initializeNodeLabels(configuration); + initializeNodeLabels(configuration, configuredNodeLabels); } - private void initializeNodeLabels(CapacitySchedulerConfiguration configuration) + private void initializeNodeLabels(CapacitySchedulerConfiguration configuration, + ConfiguredNodeLabels configuredNodeLabels) throws IOException { initializeAccessibleLabels(configuration); initializeDefaultLabelExpression(configuration); - initializeConfiguredNodeLabels(); + initializeConfiguredNodeLabels(configuration, configuredNodeLabels); validateNodeLabels(); } @@ -73,19 +72,17 @@ private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration con } } - private void initializeConfiguredNodeLabels() { - if (csContext.getCapacitySchedulerQueueManager() != null - && csContext.getCapacitySchedulerQueueManager().getConfiguredNodeLabels() != null) { + private void initializeConfiguredNodeLabels(CapacitySchedulerConfiguration configuration, + ConfiguredNodeLabels configuredNodeLabelsParam) { + if (configuredNodeLabelsParam != null) { if (queuePath.equals(ROOT)) { - this.configuredNodeLabels = csContext.getCapacitySchedulerQueueManager() - .getConfiguredNodeLabels().getAllConfiguredLabels(); + this.configuredNodeLabels = configuredNodeLabelsParam.getAllConfiguredLabels(); } else { - this.configuredNodeLabels = csContext.getCapacitySchedulerQueueManager() - .getConfiguredNodeLabels().getLabelsByQueue(queuePath); + this.configuredNodeLabels = configuredNodeLabelsParam.getLabelsByQueue(queuePath); } } else { // Fallback to suboptimal but correct logic - this.configuredNodeLabels = csContext.getConfiguration().getConfiguredNodeLabels(queuePath); + this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java index 4208bf06a24a98..38ee4d237a6f0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java @@ -31,20 +31,17 @@ * */ public class ReservationQueue extends AbstractAutoCreatedLeafQueue { - - private static final Logger LOG = LoggerFactory - .getLogger(ReservationQueue.class); + private static final Logger LOG = + LoggerFactory.getLogger(ReservationQueue.class); private PlanQueue parent; - public ReservationQueue(CapacitySchedulerContext cs, String queueName, + public ReservationQueue(CapacitySchedulerQueueContext queueContext, String queueName, PlanQueue parent) throws IOException { - super(cs, queueName, parent, null); - super.setupQueueConfigs(cs.getClusterResource(), - cs.getConfiguration()); + super(queueContext, queueName, parent, null); + super.setupQueueConfigs(queueContext.getClusterResource(), + queueContext.getConfiguration()); - LOG.debug("Initialized ReservationQueue: name={}, fullname={}", - queueName, getQueuePath()); // the following parameters are common to all reservation in the plan updateQuotas(parent.getUserLimitForReservation(), parent.getUserLimitFactor(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java index 73aad3c177193b..e8c99408ffeecc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java @@ -58,7 +58,6 @@ public class UsersManager implements AbstractUsersManager { private final AbstractLeafQueue lQueue; private final RMNodeLabelsManager labelManager; private final ResourceCalculator resourceCalculator; - private final CapacitySchedulerContext scheduler; private Map users = new ConcurrentHashMap<>(); private ResourceUsage totalResUsageForActiveUsers = new ResourceUsage(); @@ -296,17 +295,13 @@ public void setWeight(float weight) { * Leaf Queue Object * @param labelManager * Label Manager instance - * @param scheduler - * Capacity Scheduler Context * @param resourceCalculator * rc */ public UsersManager(QueueMetrics metrics, AbstractLeafQueue lQueue, - RMNodeLabelsManager labelManager, CapacitySchedulerContext scheduler, - ResourceCalculator resourceCalculator) { + RMNodeLabelsManager labelManager, ResourceCalculator resourceCalculator) { ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.lQueue = lQueue; - this.scheduler = scheduler; this.labelManager = labelManager; this.resourceCalculator = resourceCalculator; this.qUsageRatios = new UsageRatios(); @@ -844,10 +839,8 @@ partitionResource, getUsageRatio(nodePartition), /** * Update new usage ratio. * - * @param partition - * Node partition - * @param clusterResource - * Cluster Resource + * @param partition Node partition + * @param clusterResource cluster resource */ public void updateUsageRatio(String partition, Resource clusterResource) { writeLock.lock(); @@ -1064,6 +1057,8 @@ private ResourceUsage getTotalResourceUsagePerUser(String userName) { * Name of the user * @param resource * Resource to increment/decrement + * @param clusterResource + * Cluster resource (for testing purposes only) * @param nodePartition * Node label * @param isAllocate @@ -1071,6 +1066,7 @@ private ResourceUsage getTotalResourceUsagePerUser(String userName) { * @return user */ public User updateUserResourceUsage(String userName, Resource resource, + Resource clusterResource, String nodePartition, boolean isAllocate) { this.writeLock.lock(); try { @@ -1086,7 +1082,7 @@ public User updateUserResourceUsage(String userName, Resource resource, // Update usage ratios Resource resourceByLabel = labelManager.getResourceByLabel(nodePartition, - scheduler.getClusterResource()); + clusterResource); incQueueUsageRatio(nodePartition, user.updateUsageRatio( resourceCalculator, resourceByLabel, nodePartition)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java index b396d5761de164..669435ecba0fff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java @@ -80,7 +80,7 @@ private boolean checkHeadroom(ResourceLimits currentResourceLimits, // require Resource resourceCouldBeUnReserved = application.getAppAttemptResourceUsage().getReserved(nodePartition); - if (!application.getCSLeafQueue().getReservationContinueLooking()) { + if (!application.getCSLeafQueue().isReservationsContinueLooking()) { // If we don't allow reservation continuous looking, // we won't allow to unreserve before allocation. resourceCouldBeUnReserved = Resources.none(); @@ -154,7 +154,7 @@ private ContainerAllocation preCheckForNodeCandidateSet(FiCaSchedulerNode node, return ContainerAllocation.PRIORITY_SKIPPED; } - if (!application.getCSLeafQueue().getReservationContinueLooking()) { + if (!application.getCSLeafQueue().isReservationsContinueLooking()) { if (!shouldAllocOrReserveNewContainer(schedulerKey, required)) { LOG.debug("doesn't need containers based on reservation algo!"); ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( @@ -551,7 +551,7 @@ private ContainerAllocation assignContainer(Resource clusterResource, RMContainer unreservedContainer = null; boolean reservationsContinueLooking = - application.getCSLeafQueue().getReservationContinueLooking(); + application.getCSLeafQueue().isReservationsContinueLooking(); // Check if we need to kill some containers to allocate this one List toKillContainers = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java index 7458df904518fc..14d3555e100a87 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java @@ -70,7 +70,6 @@ public class GuaranteedOrZeroCapacityOverTimePolicy implements AutoCreatedQueueManagementPolicy { private static final int DEFAULT_QUEUE_PRINT_SIZE_LIMIT = 25; - private CapacitySchedulerContext scheduler; private ManagedParentQueue managedParentQueue; private static final Logger LOG = @@ -263,9 +262,9 @@ private class PendingApplicationComparator @Override public int compare(FiCaSchedulerApp app1, FiCaSchedulerApp app2) { - RMApp rmApp1 = scheduler.getRMContext().getRMApps().get( + RMApp rmApp1 = managedParentQueue.getQueueContext().getRMApp( app1.getApplicationId()); - RMApp rmApp2 = scheduler.getRMContext().getRMApps().get( + RMApp rmApp2 = managedParentQueue.getQueueContext().getRMApp( app2.getApplicationId()); if (rmApp1 != null && rmApp2 != null) { return Long.compare(rmApp1.getSubmitTime(), rmApp2.getSubmitTime()); @@ -283,10 +282,7 @@ public int compare(FiCaSchedulerApp app1, FiCaSchedulerApp app2) { new PendingApplicationComparator(); @Override - public void init(final CapacitySchedulerContext schedulerContext, - final ParentQueue parentQueue) throws IOException { - this.scheduler = schedulerContext; - + public void init(final ParentQueue parentQueue) throws IOException { ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); writeLock = lock.writeLock(); @@ -372,7 +368,7 @@ public List computeQueueManagementChanges() //Populate new entitlements return leafQueueEntitlements.mapToQueueManagementChanges((leafQueueName, capacities) -> { AutoCreatedLeafQueue leafQueue = - (AutoCreatedLeafQueue) scheduler.getCapacitySchedulerQueueManager() + (AutoCreatedLeafQueue) managedParentQueue.getQueueContext().getQueueManager() .getQueue(leafQueueName); AutoCreatedLeafQueueConfig newTemplate = buildTemplate(capacities); return new QueueManagementChange.UpdateQueue(leafQueue, newTemplate); @@ -651,7 +647,8 @@ public void commitQueueManagementChanges( .mergeCapacities(updatedQueueTemplate.getQueueCapacities()); leafQueue.getQueueResourceQuotas() .setConfiguredMinResource(Resources.multiply( - this.scheduler.getClusterResource(), updatedQueueTemplate + managedParentQueue.getQueueContext().getClusterResource(), + updatedQueueTemplate .getQueueCapacities().getCapacity(nodeLabel))); deactivate(leafQueue, nodeLabel); } @@ -693,8 +690,7 @@ public boolean hasPendingApps(final AutoCreatedLeafQueue leafQueue) { } @Override - public void reinitialize(CapacitySchedulerContext schedulerContext, - final ParentQueue parentQueue) throws IOException { + public void reinitialize(final ParentQueue parentQueue) throws IOException { if (!(parentQueue instanceof ManagedParentQueue)) { throw new IllegalStateException( "Expected instance of type " + ManagedParentQueue.class + " found " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 4a393ccf0cb539..87147ce62c9a5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -388,7 +388,7 @@ public void testQueueSubmitWithACLsEnabledWithQueueMappingForAutoCreatedQueue() MockRM newMockRM = new MockRM(csConf); CapacityScheduler cs = ((CapacityScheduler) newMockRM.getResourceScheduler()); - ManagedParentQueue managedParentQueue = new ManagedParentQueue(cs, + ManagedParentQueue managedParentQueue = new ManagedParentQueue(cs.getQueueContext(), "managedparent", cs.getQueue("root"), null); cs.getCapacitySchedulerQueueManager().addQueue("managedparent", managedParentQueue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index 08462332818cfa..d9051dd53e1bfc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -215,7 +215,7 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue() CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue(QUEUED); - AutoCreatedLeafQueue d1 = new AutoCreatedLeafQueue(cs, "d1", parentQueue); + AutoCreatedLeafQueue d1 = new AutoCreatedLeafQueue(cs.getQueueContext(), "d1", parentQueue); cs.addQueue(d1); /** @@ -240,7 +240,7 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue() * d1 will occupy all entire resource * of Managed Parent queue. */ - AutoCreatedLeafQueue d2 = new AutoCreatedLeafQueue(cs, "d2", parentQueue); + AutoCreatedLeafQueue d2 = new AutoCreatedLeafQueue(cs.getQueueContext(), "d2", parentQueue); cs.addQueue(d2); cs.getRootQueue().updateClusterResource(cs.getClusterResource(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index a1252cfade79c9..33134babc9f57d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -107,19 +107,11 @@ public void setUp() throws IOException { rmContext = TestUtils.getMockRMContext(); Resource clusterResource = Resources.createResource(10 * 16 * GB, 10 * 32); - CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class); - when(csContext.getConfiguration()).thenReturn(csConf); - when(csContext.getConf()).thenReturn(conf); - when(csContext.getMinimumResourceCapability()). - thenReturn(Resources.createResource(GB, 1)); - when(csContext.getMaximumResourceCapability()). - thenReturn(Resources.createResource(16*GB, 32)); - when(csContext.getClusterResource()). - thenReturn(clusterResource); - when(csContext.getResourceCalculator()). - thenReturn(resourceCalculator); + CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator, + Resources.createResource(GB, 1), Resources.createResource(16*GB, 32), + clusterResource); when(csContext.getRMContext()).thenReturn(rmContext); - when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); + CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext); RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf); @@ -129,13 +121,13 @@ public void setUp() throws IOException { CSQueueStore queues = new CSQueueStore(); root = CapacitySchedulerQueueManager - .parseQueue(csContext, csConf, null, "root", + .parseQueue(queueContext, csConf, null, "root", queues, queues, TestUtils.spyHook); root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); - queue = spy(new LeafQueue(csContext, A, root, null)); + queue = spy(new LeafQueue(queueContext, A, root, null)); QueueResourceQuotas queueResourceQuotas = ((LeafQueue) queues.get(A)) .getQueueResourceQuotas(); doReturn(queueResourceQuotas).when(queue).getQueueResourceQuotas(); @@ -278,28 +270,21 @@ public void testLimitsComputation() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(csConf); - YarnConfiguration conf = new YarnConfiguration(); - - CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class); - when(csContext.getConfiguration()).thenReturn(csConf); - when(csContext.getConf()).thenReturn(conf); - when(csContext.getMinimumResourceCapability()). - thenReturn(Resources.createResource(GB, 1)); - when(csContext.getMaximumResourceCapability()). - thenReturn(Resources.createResource(16*GB, 16)); - when(csContext.getResourceCalculator()).thenReturn(resourceCalculator); - when(csContext.getRMContext()).thenReturn(rmContext); - when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); - + // Say cluster has 100 nodes of 16G each Resource clusterResource = Resources.createResource(100 * 16 * GB, 100 * 16); - when(csContext.getClusterResource()).thenReturn(clusterResource); + + CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator, Resources.createResource(GB, 1), + Resources.createResource(16*GB, 16), clusterResource); + CapacitySchedulerQueueManager queueManager = csContext.getCapacitySchedulerQueueManager(); + CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext); CSQueueStore queues = new CSQueueStore(); CSQueue root = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, "root", queues, queues, TestUtils.spyHook); + queueManager.setRootQueue(root); root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); @@ -367,12 +352,14 @@ public void testLimitsComputation() throws Exception { // Change the per-queue max AM resources percentage. csConf.setFloat(PREFIX + queue.getQueuePath() + ".maximum-am-resource-percent", 0.5f); + queueContext.reinitialize(); // Re-create queues to get new configs. queues = new CSQueueStore(); root = CapacitySchedulerQueueManager.parseQueue( - csContext, csConf, null, "root", + queueContext, csConf, null, "root", queues, queues, TestUtils.spyHook); clusterResource = Resources.createResource(100 * 16 * GB); + queueManager.setRootQueue(root); root.updateClusterResource(clusterResource, new ResourceLimits( clusterResource)); @@ -391,10 +378,11 @@ public void testLimitsComputation() throws Exception { // Change the per-queue max applications. csConf.setInt(PREFIX + queue.getQueuePath() + ".maximum-applications", 9999); + queueContext.reinitialize(); // Re-create queues to get new configs. queues = new CSQueueStore(); root = CapacitySchedulerQueueManager.parseQueue( - csContext, csConf, null, "root", + queueContext, csConf, null, "root", queues, queues, TestUtils.spyHook); root.updateClusterResource(clusterResource, new ResourceLimits( clusterResource)); @@ -587,26 +575,19 @@ public void testHeadroom() throws Exception { new CapacitySchedulerConfiguration(); csConf.setUserLimit(CapacitySchedulerConfiguration.ROOT + "." + A, 25); setupQueueConfiguration(csConf); - YarnConfiguration conf = new YarnConfiguration(); - - CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class); - when(csContext.getConfiguration()).thenReturn(csConf); - when(csContext.getConf()).thenReturn(conf); - when(csContext.getMinimumResourceCapability()). - thenReturn(Resources.createResource(GB)); - when(csContext.getMaximumResourceCapability()). - thenReturn(Resources.createResource(16*GB)); - when(csContext.getResourceCalculator()).thenReturn(resourceCalculator); - when(csContext.getRMContext()).thenReturn(rmContext); - when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); - + // Say cluster has 100 nodes of 16G each Resource clusterResource = Resources.createResource(100 * 16 * GB); - when(csContext.getClusterResource()).thenReturn(clusterResource); + + CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator, Resources.createResource(GB), + Resources.createResource(16*GB), clusterResource); + CapacitySchedulerQueueManager queueManager = csContext.getCapacitySchedulerQueueManager(); + CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext); CSQueueStore queues = new CSQueueStore(); - CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext, + CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, "root", queues, queues, TestUtils.spyHook); + queueManager.setRootQueue(rootQueue); rootQueue.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); @@ -952,27 +933,17 @@ public void testAMResourceLimitWithDRCAndFullParent() throws Exception { setupQueueConfiguration(csConf); csConf.setFloat(CapacitySchedulerConfiguration. MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.3f); - YarnConfiguration conf = new YarnConfiguration(); - - CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class); - when(csContext.getConfiguration()).thenReturn(csConf); - when(csContext.getConf()).thenReturn(conf); - when(csContext.getMinimumResourceCapability()). - thenReturn(Resources.createResource(GB)); - when(csContext.getMaximumResourceCapability()). - thenReturn(Resources.createResource(16*GB)); - when(csContext.getResourceCalculator()). - thenReturn(new DominantResourceCalculator()); - when(csContext.getRMContext()).thenReturn(rmContext); - when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); // Total cluster resources. Resource clusterResource = Resources.createResource(100 * GB, 1000); - when(csContext.getClusterResource()).thenReturn(clusterResource); + + CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext( + createCSContext(csConf, new DominantResourceCalculator(), Resources.createResource(GB), + Resources.createResource(16*GB), clusterResource)); // Set up queue hierarchy. CSQueueStore queues = new CSQueueStore(); - CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext, + CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, "root", queues, queues, TestUtils.spyHook); rootQueue.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); @@ -1015,4 +986,32 @@ public void testAMResourceLimitWithDRCAndFullParent() throws Exception { + amLimit.getVirtualCores(), amLimit.getVirtualCores() >= expectedAmLimit.getVirtualCores()); } + + private CapacitySchedulerContext createCSContext(CapacitySchedulerConfiguration csConf, + ResourceCalculator rc, Resource minResource, Resource maxResource, Resource clusterResource) { + YarnConfiguration conf = new YarnConfiguration(); + + CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class); + when(csContext.getConfiguration()).thenReturn(csConf); + when(csContext.getConf()).thenReturn(conf); + when(csContext.getMinimumResourceCapability()). + thenReturn(minResource); + when(csContext.getMaximumResourceCapability()). + thenReturn(maxResource); + when(csContext.getResourceCalculator()). + thenReturn(rc); + CapacitySchedulerQueueManager queueManager = new CapacitySchedulerQueueManager(conf, + rmContext.getNodeLabelManager(), null); + when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); + when(csContext.getCapacitySchedulerQueueManager()).thenReturn(queueManager); + + when(csContext.getRMContext()).thenReturn(rmContext); + when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); + + // Total cluster resources. + when(csContext.getClusterResource()).thenReturn(clusterResource); + + return csContext; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index a228d254d5ad83..4c2ec87e705299 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -777,6 +777,12 @@ public void testHeadroom() throws Exception { when(spyRMContext.getNodeLabelManager()).thenReturn(mgr); when(csContext.getRMContext()).thenReturn(spyRMContext); when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); + CapacitySchedulerQueueManager queueManager = + new CapacitySchedulerQueueManager(csConf, mgr, null); + when(csContext.getCapacitySchedulerQueueManager()).thenReturn(queueManager); + + // Setup nodelabels + queueManager.reinitConfiguredNodeLabels(csConf); mgr.activateNode(NodeId.newInstance("h0", 0), Resource.newInstance(160 * GB, 16)); // default Label @@ -789,16 +795,15 @@ public void testHeadroom() throws Exception { Resource clusterResource = Resources.createResource(160 * GB); when(csContext.getClusterResource()).thenReturn(clusterResource); + CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext); + CSQueueStore queues = new CSQueueStore(); - CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext, + CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, "root", queues, queues, TestUtils.spyHook); + queueManager.setRootQueue(rootQueue); rootQueue.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); - ResourceUsage queueResUsage = rootQueue.getQueueResourceUsage(); - when(csContext.getClusterResourceUsage()) - .thenReturn(queueResUsage); - // Manipulate queue 'a' LeafQueue queue = TestLeafQueue.stubLeafQueue((LeafQueue) queues.get("b2")); queue.updateClusterResource(clusterResource, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java index b560d9798e2a5d..f7460de7aafd6f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java @@ -92,9 +92,13 @@ public void setup() throws IOException { when(preemptionManager.getKillableResource(any(), anyString())) .thenReturn(Resource.newInstance(0, 0)); when(scheduler.getPreemptionManager()).thenReturn(preemptionManager); + when(scheduler.getActivitiesManager()).thenReturn(activitiesManager); queueManager = new CapacitySchedulerQueueManager(csConfig, labelManager, appPriorityACLManager); queueManager.setCapacitySchedulerContext(scheduler); + when(scheduler.getCapacitySchedulerQueueManager()).thenReturn(queueManager); + CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(scheduler); + when(scheduler.getQueueContext()).thenReturn(queueContext); queueManager.initializeQueues(csConfig); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java index ad0843216d172a..8ec8d62e744546 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java @@ -39,6 +39,7 @@ public class TestCSQueueStore { private CSQueue root; private CapacitySchedulerContext csContext; + private CapacitySchedulerQueueContext queueContext; @Before public void setUp() throws IOException { @@ -62,22 +63,26 @@ public void setUp() throws IOException { when(csContext.getResourceCalculator()). thenReturn(resourceCalculator); when(csContext.getRMContext()).thenReturn(rmContext); + when(csContext.getCapacitySchedulerQueueManager()).thenReturn( + new CapacitySchedulerQueueManager(csConf, null, null)); + + queueContext = new CapacitySchedulerQueueContext(csContext); CSQueueStore queues = new CSQueueStore(); root = CapacitySchedulerQueueManager - .parseQueue(csContext, csConf, null, "root", + .parseQueue(queueContext, csConf, null, "root", queues, queues, TestUtils.spyHook); } public CSQueue createLeafQueue(String name, CSQueue parent) throws IOException { - return new LeafQueue(csContext, name, parent, null); + return new LeafQueue(queueContext, name, parent, null); } public CSQueue createParentQueue(String name, CSQueue parent) throws IOException { - return new ParentQueue(csContext, name, parent, null); + return new ParentQueue(queueContext, name, parent, null); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java index 835d95ec874187..90c63dbd2b95fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java @@ -680,7 +680,7 @@ public void testAutoCreationFailsWhenParentCapacityExceeded() // Test add one auto created queue dynamically and manually modify // capacity ManagedParentQueue parentQueue = (ManagedParentQueue) newCS.getQueue("c"); - AutoCreatedLeafQueue c1 = new AutoCreatedLeafQueue(newCS, "c1", + AutoCreatedLeafQueue c1 = new AutoCreatedLeafQueue(newCS.getQueueContext(), "c1", parentQueue); newCS.addQueue(c1); c1.setCapacity(0.5f); @@ -689,13 +689,13 @@ public void testAutoCreationFailsWhenParentCapacityExceeded() setEntitlement(c1, new QueueEntitlement(0.5f, 1f)); - AutoCreatedLeafQueue c2 = new AutoCreatedLeafQueue(newCS, "c2", + AutoCreatedLeafQueue c2 = new AutoCreatedLeafQueue(newCS.getQueueContext(), "c2", parentQueue); newCS.addQueue(c2); setEntitlement(c2, new QueueEntitlement(0.5f, 1f)); try { - AutoCreatedLeafQueue c3 = new AutoCreatedLeafQueue(newCS, "c3", + AutoCreatedLeafQueue c3 = new AutoCreatedLeafQueue(newCS.getQueueContext(), "c3", parentQueue); newCS.addQueue(c3); fail("Expected exception for auto queue creation failure"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java index 4dd537d3a0cb85..7cfd457cb76b16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java @@ -86,14 +86,14 @@ public void testRefreshQueuesWithReservations() throws Exception { // Test add one reservation dynamically and manually modify capacity ReservationQueue a1 = - new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a")); + new ReservationQueue(cs.getQueueContext(), "a1", (PlanQueue) cs.getQueue("a")); cs.addQueue(a1); a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f)); // Test add another reservation queue and use setEntitlement to modify // capacity ReservationQueue a2 = - new ReservationQueue(cs, "a2", (PlanQueue) cs.getQueue("a")); + new ReservationQueue(cs.getQueueContext(), "a2", (PlanQueue) cs.getQueue("a")); cs.addQueue(a2); cs.setEntitlement("a2", new QueueEntitlement(A2_CAPACITY / 100, 1.0f)); @@ -116,7 +116,7 @@ public void testAddQueueFailCases() throws Exception { try { // Test invalid addition (adding non-zero size queue) ReservationQueue a1 = - new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a")); + new ReservationQueue(cs.getQueueContext(), "a1", (PlanQueue) cs.getQueue("a")); a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f)); cs.addQueue(a1); fail(); @@ -126,7 +126,7 @@ public void testAddQueueFailCases() throws Exception { // Test add one reservation dynamically and manually modify capacity ReservationQueue a1 = - new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a")); + new ReservationQueue(cs.getQueueContext(), "a1", (PlanQueue) cs.getQueue("a")); cs.addQueue(a1); //set default queue capacity to zero ((ReservationQueue) cs @@ -138,7 +138,7 @@ public void testAddQueueFailCases() throws Exception { // Test add another reservation queue and use setEntitlement to modify // capacity ReservationQueue a2 = - new ReservationQueue(cs, "a2", (PlanQueue) cs.getQueue("a")); + new ReservationQueue(cs.getQueueContext(), "a2", (PlanQueue) cs.getQueue("a")); cs.addQueue(a2); @@ -165,7 +165,7 @@ public void testRemoveQueue() throws Exception { // Test add one reservation dynamically and manually modify capacity ReservationQueue a1 = - new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a")); + new ReservationQueue(cs.getQueueContext(), "a1", (PlanQueue) cs.getQueue("a")); cs.addQueue(a1); a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f)); @@ -249,7 +249,7 @@ public void testMoveAppToPlanQueue() throws Exception { // create the default reservation queue String defQName = "a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX; ReservationQueue defQ = - new ReservationQueue(scheduler, defQName, + new ReservationQueue(scheduler.getQueueContext(), defQName, (PlanQueue) scheduler.getQueue("a")); scheduler.addQueue(defQ); defQ.setEntitlement(new QueueEntitlement(1f, 1f)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java index b71fe063927ac8..6c84c8eab5ef15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java @@ -177,7 +177,7 @@ private void testUserLimitThroughputWithNumberOfResourceTypes( LeafQueue qb = (LeafQueue)cs.getQueue(queueName); // For now make user limit large so we can activate all applications qb.setUserLimitFactor((float)100.0); - qb.setupConfigurableCapacities(); + qb.setupConfigurableCapacities(cs.getConfiguration()); lqs[i] = qb; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java index 31ad107de94c3b..1af3563c52729d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java @@ -74,6 +74,7 @@ public class TestChildQueueOrder { YarnConfiguration conf; CapacitySchedulerConfiguration csConf; CapacitySchedulerContext csContext; + CapacitySchedulerQueueContext queueContext; final static int GB = 1024; final static String DEFAULT_RACK = "/default"; @@ -100,6 +101,10 @@ public void setUp() throws Exception { thenReturn(resourceComparator); when(csContext.getRMContext()).thenReturn(rmContext); when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); + when(csContext.getCapacitySchedulerQueueManager()).thenReturn( + new CapacitySchedulerQueueManager(csConf, rmContext.getNodeLabelManager(), null)); + + queueContext = new CapacitySchedulerQueueContext(csContext); } private FiCaSchedulerApp getMockApplication(int appId, String user) { @@ -219,9 +224,10 @@ private void setupSortedQueues(CapacitySchedulerConfiguration conf) { public void testSortedQueues() throws Exception { // Setup queue configs setupSortedQueues(csConf); + queueContext.reinitialize(); CSQueueStore queues = new CSQueueStore(); CSQueue root = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index d3545bd41f57a1..1da7ce18ee01cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -143,6 +143,7 @@ public class TestLeafQueue { CapacityScheduler cs; CapacitySchedulerConfiguration csConf; CapacitySchedulerContext csContext; + CapacitySchedulerQueueContext queueContext; private RMApp rmApp; CSQueue root; @@ -203,6 +204,7 @@ private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels) csConf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true); csConf.setBoolean(CapacitySchedulerConfiguration.RESERVE_CONT_LOOK_ALL_NODES, false); + csConf.setResourceComparator(rC.getClass()); final String newRoot = "root" + System.currentTimeMillis(); setupQueueConfiguration(csConf, newRoot, withNodeLabels); YarnConfiguration conf = new YarnConfiguration(); @@ -228,12 +230,20 @@ private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels) containerTokenSecretManager.rollMasterKey(); when(csContext.getContainerTokenSecretManager()).thenReturn( containerTokenSecretManager); + CapacitySchedulerQueueManager queueManager = + new CapacitySchedulerQueueManager(csConf, null, null); + when(csContext.getCapacitySchedulerQueueManager()).thenReturn(queueManager); + + queueManager.reinitConfiguredNodeLabels(csConf); + + queueContext = new CapacitySchedulerQueueContext(csContext); root = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, ROOT, queues, queues, TestUtils.spyHook); + queueManager.setRootQueue(root); root.updateClusterResource(Resources.createResource(100 * 16 * GB, 100 * 32), new ResourceLimits(Resources.createResource(100 * 16 * GB, 100 * 32))); @@ -242,8 +252,8 @@ private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels) .thenReturn(queueResUsage); cs.setRMContext(spyRMContext); - cs.init(csConf); cs.setResourceCalculator(rC); + cs.init(csConf); when(spyRMContext.getScheduler()).thenReturn(cs); when(spyRMContext.getYarnConfiguration()) @@ -1087,11 +1097,12 @@ public void testUserLimitCache() throws Exception { csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 100); csConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 100); + queueContext.reinitialize(); // reinitialize queues CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); @@ -1305,11 +1316,12 @@ public void testUserLimitCacheActiveUsersChanged() throws Exception { csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 100); csConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 100); + queueContext.reinitialize(); // reinitialize queues CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); @@ -1920,6 +1932,7 @@ public void testUserSpecificUserLimits() throws Exception { + CapacitySchedulerConfiguration.USER_WEIGHT, 0.7f); csConf.reinitializeConfigurationProperties(); + queueContext.reinitialize(); when(csContext.getClusterResource()) .thenReturn(Resources.createResource(16 * GB, 32)); @@ -3216,10 +3229,12 @@ public void testRackLocalityDelayScheduling() throws Exception { csConf.setInt(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, 2); csConf.setInt( CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 1); + queueContext.reinitialize(); CSQueueStore newQueues = new CSQueueStore(); - CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, + CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, ROOT, newQueues, queues, TestUtils.spyHook); + csContext.getCapacitySchedulerQueueManager().setRootQueue(newRoot); root.reinitialize(newRoot, cs.getClusterResource()); // Manipulate queue 'b' @@ -3652,9 +3667,10 @@ public void testActivateApplicationAfterQueueRefresh() throws Exception { CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2); + queueContext.reinitialize(); CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, ROOT, newQueues, queues, TestUtils.spyHook); @@ -3683,12 +3699,14 @@ public void testLocalityDelaysAfterQueueRefresh() throws Exception { csConf.setInt(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, 60); csConf.setInt( CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 600); + queueContext.reinitialize(); CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, ROOT, newQueues, queues, TestUtils.spyHook); + csContext.getCapacitySchedulerQueueManager().setRootQueue(newRoot); root.reinitialize(newRoot, cs.getClusterResource()); // after reinitialization @@ -4043,8 +4061,14 @@ public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f); + CapacitySchedulerQueueManager queueManager = new CapacitySchedulerQueueManager(csConf, + rmContext.getNodeLabelManager(), null); + when(csContext.getCapacitySchedulerQueueManager()).thenReturn(queueManager); + + CapacitySchedulerQueueContext newQueueContext = new CapacitySchedulerQueueContext(csContext); + CSQueue root; - root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + root = CapacitySchedulerQueueManager.parseQueue(newQueueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); @@ -4060,9 +4084,10 @@ public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() csConf.setFloat( CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f); + newQueueContext.reinitialize(); clusterResource = Resources.createResource(100 * 20 * GB, 100 * 32); CSQueueStore newQueues = new CSQueueStore(); - CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, + CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(newQueueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); root.reinitialize(newRoot, clusterResource); @@ -5112,15 +5137,15 @@ public void testSetupQueueConfigsWithSpecifiedConfiguration() assertEquals(0, conf.size()); conf.setNodeLocalityDelay(60); - conf.setCapacity(ROOT + DOT + leafQueueName, 10); - conf.setMaximumCapacity(ROOT + DOT + leafQueueName, 100); - conf.setUserLimitFactor(ROOT + DOT +leafQueueName, 0.1f); + csConf.setCapacity(ROOT + DOT + leafQueueName, 10); + csConf.setMaximumCapacity(ROOT + DOT + leafQueueName, 100); + csConf.setUserLimitFactor(ROOT + DOT +leafQueueName, 0.1f); csConf.setNodeLocalityDelay(30); csConf.setGlobalMaximumApplicationsPerQueue(20); + queueContext.reinitialize(); - LeafQueue leafQueue = new LeafQueue(csContext, conf, - leafQueueName, cs.getRootQueue(), + LeafQueue leafQueue = new LeafQueue(queueContext, leafQueueName, cs.getRootQueue(), null); leafQueue.updateClusterResource(Resource.newInstance(0, 0), @@ -5148,6 +5173,7 @@ public void testSetupQueueConfigsWithSpecifiedConfiguration() // limit maximum apps by max system apps csConf.setMaximumSystemApplications(15); + queueContext.reinitialize(); leafQueue.updateClusterResource(Resource.newInstance(0, 0), new ResourceLimits(Resource.newInstance(0, 0))); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 31ece4f5f0f393..476abc638fba74 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -79,7 +79,8 @@ public class TestParentQueue { YarnConfiguration conf; CapacitySchedulerConfiguration csConf; CapacitySchedulerContext csContext; - + CapacitySchedulerQueueContext queueContext; + final static int GB = 1024; final static String DEFAULT_RACK = "/default"; @@ -105,6 +106,10 @@ public void setUp() throws Exception { when(csContext.getResourceCalculator()). thenReturn(resourceComparator); when(csContext.getRMContext()).thenReturn(rmContext); + when(csContext.getCapacitySchedulerQueueManager()).thenReturn( + new CapacitySchedulerQueueManager(csConf, rmContext.getNodeLabelManager(), null)); + + queueContext = new CapacitySchedulerQueueContext(csContext); } private static final String A = "a"; @@ -121,6 +126,8 @@ private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) { conf.setCapacity(Q_A, 30); conf.setCapacity(Q_B, 70); + + queueContext.reinitialize(); LOG.info("Setup top-level queues a and b"); } @@ -137,6 +144,8 @@ private void setupSingleLevelQueuesWithAbsoluteResource( conf.setMinimumResourceRequirement("", new QueuePath(Q_B), QUEUE_B_RESOURCE); + queueContext.reinitialize(); + LOG.info("Setup top-level queues a and b with absolute resource"); } @@ -253,7 +262,7 @@ public void testSingleLevelQueues() throws Exception { CSQueueStore queues = new CSQueueStore(); CSQueue root = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); @@ -371,11 +380,12 @@ public void testSingleLevelQueuesPrecision() throws Exception { setupSingleLevelQueues(csConf); csConf.setCapacity(Q_A, 30); csConf.setCapacity(Q_B, 70.5F); + queueContext.reinitialize(); CSQueueStore queues = new CSQueueStore(); boolean exceptionOccurred = false; try { - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); } catch (IOException ie) { @@ -386,10 +396,11 @@ public void testSingleLevelQueuesPrecision() throws Exception { } csConf.setCapacity(Q_A, 30); csConf.setCapacity(Q_B, 70); + queueContext.reinitialize(); exceptionOccurred = false; queues.clear(); try { - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); } catch (IllegalArgumentException ie) { @@ -400,10 +411,11 @@ public void testSingleLevelQueuesPrecision() throws Exception { } csConf.setCapacity(Q_A, 30); csConf.setCapacity(Q_B, 70.005F); + queueContext.reinitialize(); exceptionOccurred = false; queues.clear(); try { - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); } catch (IllegalArgumentException ie) { @@ -470,6 +482,7 @@ private void setupMultiLevelQueues(CapacitySchedulerConfiguration conf) { conf.setQueues(Q_C111, new String[] {C1111}); final String Q_C1111= Q_C111 + "." + C1111; conf.setCapacity(Q_C1111, 100); + queueContext.reinitialize(); } @Test @@ -495,7 +508,7 @@ public void testMultiLevelQueues() throws Exception { CSQueueStore queues = new CSQueueStore(); CSQueue root = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); @@ -657,9 +670,10 @@ public void testQueueCapacitySettingChildZero() throws Exception { csConf.setCapacity(Q_B + "." + B1, 0); csConf.setCapacity(Q_B + "." + B2, 0); csConf.setCapacity(Q_B + "." + B3, 0); + queueContext.reinitialize(); CSQueueStore queues = new CSQueueStore(); - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); } @@ -672,9 +686,10 @@ public void testQueueCapacitySettingParentZero() throws Exception { // set parent capacity to 0 when child not 0 csConf.setCapacity(Q_B, 0); csConf.setCapacity(Q_A, 60); + queueContext.reinitialize(); CSQueueStore queues = new CSQueueStore(); - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); } @@ -690,8 +705,9 @@ public void testQueueCapacitySettingParentZeroChildren100pctZeroSumAllowed() csConf.setCapacity(Q_B, 0); csConf.setCapacity(Q_A, 60); csConf.setAllowZeroCapacitySum(Q_B, true); + queueContext.reinitialize(); CSQueueStore queues = new CSQueueStore(); - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); } @@ -710,8 +726,9 @@ public void testQueueCapacitySettingParentZeroChildren50pctZeroSumAllowed() csConf.setCapacity(Q_B + "." + B2, 20); csConf.setCapacity(Q_B + "." + B3, 20); csConf.setAllowZeroCapacitySum(Q_B, true); + queueContext.reinitialize(); CSQueueStore queues = new CSQueueStore(); - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); } @@ -730,8 +747,9 @@ public void testQueueCapacitySettingParentNonZeroChildrenZeroSumAllowed() csConf.setCapacity(Q_B + "." + B2, 0); csConf.setCapacity(Q_B + "." + B3, 0); csConf.setAllowZeroCapacitySum(Q_B, true); + queueContext.reinitialize(); CSQueueStore queues = new CSQueueStore(); - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); } @@ -746,12 +764,12 @@ public void testQueueCapacityZero() throws Exception { csConf.setCapacity(Q_B + "." + B1, 0); csConf.setCapacity(Q_B + "." + B2, 0); csConf.setCapacity(Q_B + "." + B3, 0); - csConf.setCapacity(Q_A, 60); + queueContext.reinitialize(); CSQueueStore queues = new CSQueueStore(); try { - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); } catch (IllegalArgumentException e) { @@ -767,7 +785,7 @@ public void testOffSwitchScheduling() throws Exception { CSQueueStore queues = new CSQueueStore(); CSQueue root = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); @@ -849,7 +867,7 @@ public void testOffSwitchSchedulingMultiLevelQueues() throws Exception { //B3 CSQueueStore queues = new CSQueueStore(); CSQueue root = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); @@ -948,10 +966,11 @@ public void testQueueAcl() throws Exception { csConf.setAcl(Q_C, QueueACL.ADMINISTER_QUEUE, "*"); final String Q_C11= Q_C + "." + C1 + "." + C11; csConf.setAcl(Q_C11, QueueACL.SUBMIT_APPLICATIONS, "*"); + queueContext.reinitialize(); CSQueueStore queues = new CSQueueStore(); CSQueue root = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); YarnAuthorizationProvider authorizer = @@ -1014,7 +1033,7 @@ public void testAbsoluteResourceWithChangeInClusterResource() setupSingleLevelQueuesWithAbsoluteResource(csConf); CSQueueStore queues = new CSQueueStore(); - CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, + CSQueue root = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); @@ -1085,7 +1104,7 @@ public void testDeriveCapacityFromAbsoluteConfigurations() throws Exception { setupSingleLevelQueuesWithAbsoluteResource(csConf); CSQueueStore queues = new CSQueueStore(); - CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, + CSQueue root = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); @@ -1138,6 +1157,7 @@ public void testDeriveCapacityFromAbsoluteConfigurations() throws Exception { // Set GlobalMaximumApplicationsPerQueue in csConf csConf.setGlobalMaximumApplicationsPerQueue(8000); + queueContext.reinitialize(); root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); @@ -1155,6 +1175,7 @@ public void testDeriveCapacityFromAbsoluteConfigurations() throws Exception { Integer.toString(queueAMaxApplications)); csConf.set("yarn.scheduler.capacity." + Q_B + ".maximum-applications", Integer.toString(queueBMaxApplications)); + queueContext.reinitialize(); root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java index 8407922b6331d3..94d800732fb51e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java @@ -46,6 +46,7 @@ public class TestReservationQueue { private CapacitySchedulerConfiguration csConf; private CapacitySchedulerContext csContext; + private CapacitySchedulerQueueContext queueContext; final static int DEF_MAX_APPS = 10000; final static int GB = 1024; private final ResourceCalculator resourceCalculator = @@ -63,7 +64,7 @@ public void setup() throws IOException, SchedulerDynamicEditException { CapacitySchedulerQueueManager csQm = mock( CapacitySchedulerQueueManager.class); ConfiguredNodeLabels labels = new ConfiguredNodeLabels(csConf); - when(csQm.getConfiguredNodeLabels()).thenReturn(labels); + when(csQm.getConfiguredNodeLabelsForAllQueues()).thenReturn(labels); when(csContext.getConfiguration()).thenReturn(csConf); when(csContext.getCapacitySchedulerQueueManager()).thenReturn(csQm); when(csContext.getConf()).thenReturn(conf); @@ -78,9 +79,11 @@ public void setup() throws IOException, SchedulerDynamicEditException { RMContext mockRMContext = TestUtils.getMockRMContext(); when(csContext.getRMContext()).thenReturn(mockRMContext); + queueContext = new CapacitySchedulerQueueContext(csContext); + // create a queue - planQueue = new PlanQueue(csContext, "root", null, null); - autoCreatedLeafQueue = new ReservationQueue(csContext, "a", planQueue); + planQueue = new PlanQueue(queueContext, "root", null, null); + autoCreatedLeafQueue = new ReservationQueue(queueContext, "a", planQueue); planQueue.addChildQueue(autoCreatedLeafQueue); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index 1168f648024b10..5662df4c510371 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -90,6 +90,7 @@ public class TestReservations { CapacityScheduler cs; // CapacitySchedulerConfiguration csConf; CapacitySchedulerContext csContext; + CapacitySchedulerQueueContext queueContext; private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); @@ -135,7 +136,10 @@ private void setup(CapacitySchedulerConfiguration csConf, when(csContext.getClusterResource()).thenReturn( Resources.createResource(100 * 16 * GB, 100 * 12)); when(csContext.getResourceCalculator()).thenReturn(resourceCalculator); + CapacitySchedulerQueueManager queueManager = new CapacitySchedulerQueueManager(conf, + rmContext.getNodeLabelManager(), null); when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); + when(csContext.getCapacitySchedulerQueueManager()).thenReturn(queueManager); when(csContext.getRMContext()).thenReturn(rmContext); RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager( conf); @@ -143,12 +147,11 @@ private void setup(CapacitySchedulerConfiguration csConf, when(csContext.getContainerTokenSecretManager()).thenReturn( containerTokenSecretManager); - root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, - CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); + queueContext = new CapacitySchedulerQueueContext(csContext); - ResourceUsage queueResUsage = root.getQueueResourceUsage(); - when(csContext.getClusterResourceUsage()) - .thenReturn(queueResUsage); + root = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, + CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); + queueManager.setRootQueue(root); spyRMContext = spy(rmContext); when(spyRMContext.getScheduler()).thenReturn(cs); @@ -1181,23 +1184,24 @@ public void testAssignToQueue() throws Exception { public void refreshQueuesTurnOffReservationsContLook(LeafQueue a, CapacitySchedulerConfiguration csConf) throws Exception { // before reinitialization - assertEquals(true, a.getReservationContinueLooking()); + assertEquals(true, a.isReservationsContinueLooking()); assertEquals(true, - ((ParentQueue) a.getParent()).getReservationContinueLooking()); + ((ParentQueue) a.getParent()).isReservationsContinueLooking()); csConf.setBoolean( CapacitySchedulerConfiguration.RESERVE_CONT_LOOK_ALL_NODES, false); CSQueueStore newQueues = new CSQueueStore(); - CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, + queueContext.reinitialize(); + CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); queues = newQueues; root.reinitialize(newRoot, cs.getClusterResource()); // after reinitialization - assertEquals(false, a.getReservationContinueLooking()); + assertEquals(false, a.isReservationsContinueLooking()); assertEquals(false, - ((ParentQueue) a.getParent()).getReservationContinueLooking()); + ((ParentQueue) a.getParent()).isReservationsContinueLooking()); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUsersManager.java index 5b79ee2e255c11..c71d862a32a021 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUsersManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUsersManager.java @@ -58,15 +58,11 @@ public class TestUsersManager { @Mock private QueueMetrics metrics; - @Mock - private CapacitySchedulerContext context; - @Before public void setup() { usersManager = new UsersManager(metrics, lQueue, labelMgr, - context, new DefaultResourceCalculator()); when(lQueue.getMinimumAllocation()).thenReturn(MINIMUM_ALLOCATION); From 683202438f755aff9b4667d226b7271b2789e82d Mon Sep 17 00:00:00 2001 From: litao Date: Tue, 14 Dec 2021 10:19:06 +0800 Subject: [PATCH 11/33] HDFS-16327. Make DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY reconfigurable (#3716) --- .../blockmanagement/DatanodeManager.java | 11 ++++++++- .../hadoop/hdfs/server/namenode/NameNode.java | 24 ++++++++++++++----- .../namenode/TestNameNodeReconfigure.java | 19 +++++++++++++++ .../hadoop/hdfs/tools/TestDFSAdmin.java | 4 +++- 4 files changed, 50 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 91cd68d06547d8..ef51c6ca074ab8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -214,7 +214,7 @@ public class DatanodeManager { private static Set slowNodesUuidSet = Sets.newConcurrentHashSet(); private Daemon slowPeerCollectorDaemon; private final long slowPeerCollectionInterval; - private final int maxSlowPeerReportNodes; + private volatile int maxSlowPeerReportNodes; @Nullable private final SlowDiskTracker slowDiskTracker; @@ -515,6 +515,15 @@ public boolean getEnableAvoidSlowDataNodesForRead() { return this.avoidSlowDataNodesForRead; } + public void setMaxSlowpeerCollectNodes(int maxNodes) { + this.maxSlowPeerReportNodes = maxNodes; + } + + @VisibleForTesting + public int getMaxSlowpeerCollectNodes() { + return this.maxSlowPeerReportNodes; + } + /** * Sort the non-striped located blocks by the distance to the target host. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 1bc8b11b665b4b..8cd5d252473431 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -191,6 +191,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_DEFAULT; import static org.apache.hadoop.util.ExitUtil.terminate; import static org.apache.hadoop.util.ToolRunner.confirmPrompt; @@ -334,7 +336,8 @@ public enum OperationCategory { DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, DFS_IMAGE_PARALLEL_LOAD_KEY, DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY, - DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY)); + DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, + DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY)); private static final String USAGE = "Usage: hdfs namenode [" + StartupOption.BACKUP.getName() + "] | \n\t[" @@ -2204,7 +2207,8 @@ protected String reconfigurePropertyImpl(String property, String newVal) } else if (property.equals(DFS_IMAGE_PARALLEL_LOAD_KEY)) { return reconfigureParallelLoad(newVal); } else if (property.equals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY) - || (property.equals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY))) { + || (property.equals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY)) + || (property.equals(DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY))) { return reconfigureSlowNodesParameters(datanodeManager, property, newVal); } else { throw new ReconfigurationException(property, newVal, getConf().get( @@ -2396,24 +2400,32 @@ String reconfigureSlowNodesParameters(final DatanodeManager datanodeManager, final String property, final String newVal) throws ReconfigurationException { BlockManager bm = namesystem.getBlockManager(); namesystem.writeLock(); - boolean enable; + String result; try { if (property.equals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY)) { - enable = (newVal == null ? DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT : + boolean enable = (newVal == null ? DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT : Boolean.parseBoolean(newVal)); + result = Boolean.toString(enable); datanodeManager.setAvoidSlowDataNodesForReadEnabled(enable); } else if (property.equals( DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY)) { - enable = (newVal == null ? + boolean enable = (newVal == null ? DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT : Boolean.parseBoolean(newVal)); + result = Boolean.toString(enable); bm.setExcludeSlowNodesEnabled(enable); + } else if (property.equals(DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY)) { + int maxSlowpeerCollectNodes = (newVal == null ? + DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_DEFAULT : + Integer.parseInt(newVal)); + result = Integer.toString(maxSlowpeerCollectNodes); + datanodeManager.setMaxSlowpeerCollectNodes(maxSlowpeerCollectNodes); } else { throw new IllegalArgumentException("Unexpected property " + property + " in reconfigureSlowNodesParameters"); } LOG.info("RECONFIGURE* changed {} to {}", property, newVal); - return Boolean.toString(enable); + return result; } catch (IllegalArgumentException e) { throw new ReconfigurationException(property, newVal, getConf().get( property), e); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java index da9b4479b59ee4..fe555532cbc990 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java @@ -55,6 +55,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DEFAULT; public class TestNameNodeReconfigure { @@ -430,6 +431,24 @@ public void testEnableSlowNodesParametersAfterReconfigured() getExcludeSlowNodesEnabled(BlockType.STRIPED)); } + @Test + public void testReconfigureMaxSlowpeerCollectNodes() + throws ReconfigurationException { + final NameNode nameNode = cluster.getNameNode(); + final DatanodeManager datanodeManager = nameNode.namesystem + .getBlockManager().getDatanodeManager(); + + // By default, DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY is 5. + assertEquals(5, datanodeManager.getMaxSlowpeerCollectNodes()); + + // Reconfigure. + nameNode.reconfigureProperty( + DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY, Integer.toString(10)); + + // Assert DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY is 10. + assertEquals(10, datanodeManager.getMaxSlowpeerCollectNodes()); + } + @After public void shutDown() throws IOException { if (cluster != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java index 6cbcc35cf430bb..351d883ab56fb0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java @@ -90,6 +90,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY; import static org.apache.hadoop.hdfs.client.HdfsAdmin.TRASH_PERMISSION; import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.CoreMatchers.anyOf; @@ -432,7 +433,7 @@ public void testNameNodeGetReconfigurableProperties() throws IOException { final List outs = Lists.newArrayList(); final List errs = Lists.newArrayList(); getReconfigurableProperties("namenode", address, outs, errs); - assertEquals(15, outs.size()); + assertEquals(16, outs.size()); assertEquals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, outs.get(1)); assertEquals(DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, outs.get(2)); assertEquals(DFS_HEARTBEAT_INTERVAL_KEY, outs.get(3)); @@ -440,6 +441,7 @@ public void testNameNodeGetReconfigurableProperties() throws IOException { assertEquals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY, outs.get(5)); assertEquals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, outs.get(6)); assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(7)); + assertEquals(DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY, outs.get(8)); assertEquals(errs.size(), 0); } From 69eb1f4976fbfe3e97cb4bc35265a631aa83c99d Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Tue, 14 Dec 2021 13:51:51 +0530 Subject: [PATCH 12/33] HDFS-16373. Fix MiniDFSCluster restart in case of multiple namenodes. (#3756) Reviewed-by: Viraj Jasani Reviewed-by: litao Signed-off-by: Takanobu Asanuma --- .../apache/hadoop/hdfs/MiniDFSCluster.java | 43 ++++++++++--------- .../hadoop/hdfs/TestMiniDFSCluster.java | 8 ++++ 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index e4b6434b4860a3..703111ed2f2258 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -2267,9 +2267,11 @@ public synchronized void restartNameNode(int nnIndex, boolean waitActive, info.nameNode = nn; info.setStartOpt(startOpt); if (waitActive) { - waitClusterUp(); + if (numDataNodes > 0) { + waitNameNodeUp(nnIndex); + } LOG.info("Restarted the namenode"); - waitActive(); + waitActive(nnIndex); } } @@ -2775,11 +2777,25 @@ public void waitActive(int nnIndex) throws IOException { DFSClient client = new DFSClient(addr, conf); // ensure all datanodes have registered and sent heartbeat to the namenode - while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE), addr)) { + int failedCount = 0; + while (true) { try { - LOG.info("Waiting for cluster to become active"); - Thread.sleep(100); + while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE), addr)) { + LOG.info("Waiting for cluster to become active"); + Thread.sleep(100); + } + break; + } catch (IOException e) { + failedCount++; + // Cached RPC connection to namenode, if any, is expected to fail once + if (failedCount > 1) { + LOG.warn("Tried waitActive() " + failedCount + + " time(s) and failed, giving up. " + StringUtils + .stringifyException(e)); + throw e; + } } catch (InterruptedException e) { + throw new IOException(e); } } @@ -2815,22 +2831,7 @@ public Boolean get() { */ public void waitActive() throws IOException { for (int index = 0; index < namenodes.size(); index++) { - int failedCount = 0; - while (true) { - try { - waitActive(index); - break; - } catch (IOException e) { - failedCount++; - // Cached RPC connection to namenode, if any, is expected to fail once - if (failedCount > 1) { - LOG.warn("Tried waitActive() " + failedCount - + " time(s) and failed, giving up. " - + StringUtils.stringifyException(e)); - throw e; - } - } - } + waitActive(index); } LOG.info("Cluster is active"); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java index 74cfe9d673cb60..737795b88d4d10 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java @@ -309,6 +309,14 @@ public void testSetUpFederatedCluster() throws Exception { DFSUtil.addKeySuffixes( DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn1"))); } + + // Shutdown namenodes individually. + cluster.shutdownNameNode(0); + cluster.shutdownNameNode(1); + + // Restart namenodes individually with wait active, both should be successful. + cluster.restartNameNode(0); + cluster.restartNameNode(1); } } } From e26c0449c4fa3e74c3fba97feba36223e733d3a1 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 14 Dec 2021 17:15:12 +0800 Subject: [PATCH 13/33] HDFS-16014: Fix an issue in checking native pmdk lib by 'hadoop checknative' command (#3762) --- .../org/apache/hadoop/io/nativeio/NativeIO.java | 2 +- .../src/org/apache/hadoop/io/nativeio/pmdk_load.c | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java index 79b489b3d1a987..ebe7f213ceeb15 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java @@ -141,7 +141,7 @@ public String getMessage() { } } - // Denotes the state of supporting PMDK. The value is set by JNI. + // Denotes the state of supporting PMDK. The actual value is set via JNI. private static SupportState pmdkSupportState = SupportState.UNSUPPORTED; diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c index 502508cbf3b863..f1a1df5c9dbe32 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c @@ -35,13 +35,14 @@ #endif PmdkLibLoader * pmdkLoader; +// 1 represents loaded. Otherwise, not loaded. +int pmdkLoaded; /** * pmdk_load.c * Utility of loading the libpmem library and the required functions. * Building of this codes won't rely on any libpmem source codes, but running * into this will rely on successfully loading of the dynamic library. - * */ static const char* load_functions() { @@ -56,6 +57,10 @@ static const char* load_functions() { return NULL; } +/** + * It should be idempotent to call this function for checking + * whether PMDK lib is successfully loaded. + */ void load_pmdk_lib(char* err, size_t err_len) { const char* errMsg; const char* library = NULL; @@ -67,10 +72,13 @@ void load_pmdk_lib(char* err, size_t err_len) { err[0] = '\0'; - if (pmdkLoader != NULL) { + if (pmdkLoaded == 1) { return; } - pmdkLoader = calloc(1, sizeof(PmdkLibLoader)); + + if (pmdkLoader == NULL) { + pmdkLoader = calloc(1, sizeof(PmdkLibLoader)); + } // Load PMDK library #ifdef UNIX @@ -103,4 +111,5 @@ void load_pmdk_lib(char* err, size_t err_len) { } pmdkLoader->libname = strdup(library); + pmdkLoaded = 1; } From acb9f404773aa3cfad0b46ccead80b1320a4fd13 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Tue, 14 Dec 2021 13:41:22 +0100 Subject: [PATCH 14/33] YARN-10929. Do not use a separate config in legacy CS AQC. Contributed by Benjamin Teke --- .../scheduler/capacity/AbstractCSQueue.java | 44 +++++------- .../scheduler/capacity/AbstractLeafQueue.java | 69 ++++++++----------- .../capacity/AbstractManagedParentQueue.java | 17 +---- .../capacity/AutoCreatedLeafQueue.java | 10 +-- .../capacity/CSQueuePreemptionSettings.java | 19 ++--- .../CapacitySchedulerQueueContext.java | 4 ++ .../scheduler/capacity/LeafQueue.java | 2 +- .../capacity/ManagedParentQueue.java | 28 ++------ .../scheduler/capacity/ParentQueue.java | 10 +-- .../scheduler/capacity/PlanQueue.java | 2 +- .../capacity/QueueAllocationSettings.java | 11 +-- .../scheduler/capacity/ReservationQueue.java | 6 +- .../capacity/TestCapacitySchedulerPerf.java | 2 +- 13 files changed, 84 insertions(+), 140 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 5040b027003d1b..3a0e2ae4d838c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -173,10 +173,9 @@ private static QueuePath createQueuePath(CSQueue parent, String queueName) { return new QueuePath(parent.getQueuePath(), queueName); } - protected void setupConfigurableCapacities( - CapacitySchedulerConfiguration configuration) { + protected void setupConfigurableCapacities() { CSQueueUtils.loadCapacitiesByLabelsFromConf(queuePath, queueCapacities, - configuration, this.queueNodeLabelsSettings.getConfiguredNodeLabels()); + queueContext.getConfiguration(), this.queueNodeLabelsSettings.getConfiguredNodeLabels()); } @Override @@ -329,14 +328,14 @@ public String getDefaultNodeLabelExpression() { return this.queueNodeLabelsSettings.getDefaultLabelExpression(); } - protected void setupQueueConfigs(Resource clusterResource, - CapacitySchedulerConfiguration configuration) throws + protected void setupQueueConfigs(Resource clusterResource) throws IOException { writeLock.lock(); try { + CapacitySchedulerConfiguration configuration = queueContext.getConfiguration(); if (isDynamicQueue() || this instanceof AbstractAutoCreatedLeafQueue) { - setDynamicQueueProperties(configuration); + setDynamicQueueProperties(); } // Collect and set the Node label configuration @@ -344,7 +343,7 @@ protected void setupQueueConfigs(Resource clusterResource, getQueuePath(), queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues()); // Initialize the queue capacities - setupConfigurableCapacities(configuration); + setupConfigurableCapacities(); updateAbsoluteCapacities(); updateCapacityConfigType(); @@ -354,26 +353,23 @@ protected void setupQueueConfigs(Resource clusterResource, // Setup queue's maximumAllocation respecting the global // and the queue settings - // TODO remove the getConfiguration() param after the AQC configuration duplication - // removal is resolved - this.queueAllocationSettings.setupMaximumAllocation(configuration, - queueContext.getConfiguration(), getQueuePath(), + this.queueAllocationSettings.setupMaximumAllocation(configuration, getQueuePath(), parent); // Initialize the queue state based on previous state, configured state // and its parent state - initializeQueueState(configuration); + initializeQueueState(); authorizer = YarnAuthorizationProvider.getInstance(configuration); this.acls = configuration.getAcls(getQueuePath()); - this.userWeights = getUserWeightsFromHierarchy(configuration); + this.userWeights = getUserWeightsFromHierarchy(); this.reservationsContinueLooking = configuration.getReservationContinueLook(); - this.configuredCapacityVectors = queueContext.getConfiguration() + this.configuredCapacityVectors = configuration .parseConfiguredResourceVector(queuePath.getFullPath(), this.queueNodeLabelsSettings.getConfiguredNodeLabels()); @@ -382,10 +378,7 @@ protected void setupQueueConfigs(Resource clusterResource, this, labelManager, null); // Store preemption settings - // TODO remove the getConfiguration() param after the AQC configuration duplication - // removal is resolved - this.preemptionSettings = new CSQueuePreemptionSettings(this, configuration, - queueContext.getConfiguration()); + this.preemptionSettings = new CSQueuePreemptionSettings(this, configuration); this.priority = configuration.getQueuePriority( getQueuePath()); @@ -403,14 +396,12 @@ protected void setupQueueConfigs(Resource clusterResource, /** * Set properties specific to dynamic queues. - * @param configuration configuration on which the properties are set */ - protected void setDynamicQueueProperties( - CapacitySchedulerConfiguration configuration) { + protected void setDynamicQueueProperties() { // Set properties from parent template if (parent instanceof ParentQueue) { ((ParentQueue) parent).getAutoCreatedQueueTemplate() - .setTemplateEntriesForChild(configuration, getQueuePath()); + .setTemplateEntriesForChild(queueContext.getConfiguration(), getQueuePath()); String parentTemplate = String.format("%s.%s", parent.getQueuePath(), AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX); @@ -428,8 +419,7 @@ protected void setDynamicQueueProperties( } } - private UserWeights getUserWeightsFromHierarchy( - CapacitySchedulerConfiguration configuration) { + private UserWeights getUserWeightsFromHierarchy() { UserWeights unionInheritedWeights = UserWeights.createEmpty(); CSQueue parentQ = parent; if (parentQ != null) { @@ -439,7 +429,7 @@ private UserWeights getUserWeightsFromHierarchy( // Insert this queue's userWeights, overriding parent's userWeights if // there is an overlap. unionInheritedWeights.addFrom( - configuration.getAllUserWeightsForQueue(getQueuePath())); + queueContext.getConfiguration().getAllUserWeightsForQueue(getQueuePath())); return unionInheritedWeights; } @@ -572,9 +562,9 @@ public QueueCapacityVector getConfiguredCapacityVector( return configuredCapacityVectors.get(label); } - private void initializeQueueState(CapacitySchedulerConfiguration configuration) { + private void initializeQueueState() { QueueState previousState = getState(); - QueueState configuredState = configuration + QueueState configuredState = queueContext.getConfiguration() .getConfiguredState(getQueuePath()); QueueState parentState = (parent == null) ? null : parent.getState(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java index dff4ade9b9ef14..8b31241d527151 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java @@ -166,15 +166,12 @@ public AbstractLeafQueue(CapacitySchedulerQueueContext queueContext, } @SuppressWarnings("checkstyle:nowhitespaceafter") - protected void setupQueueConfigs(Resource clusterResource, - CapacitySchedulerConfiguration conf) throws + protected void setupQueueConfigs(Resource clusterResource) throws IOException { writeLock.lock(); try { - // TODO conf parameter can be a modified configuration with template entries and missing - // some global configs. This config duplication needs to be removed. - CapacitySchedulerConfiguration originalConfiguration = queueContext.getConfiguration(); - super.setupQueueConfigs(clusterResource, conf); + CapacitySchedulerConfiguration configuration = queueContext.getConfiguration(); + super.setupQueueConfigs(clusterResource); this.lastClusterResource = clusterResource; @@ -189,26 +186,26 @@ protected void setupQueueConfigs(Resource clusterResource, setQueueResourceLimitsInfo(clusterResource); setOrderingPolicy( - conf.getAppOrderingPolicy(getQueuePath())); + configuration.getAppOrderingPolicy(getQueuePath())); - usersManager.setUserLimit(conf.getUserLimit(getQueuePath())); - usersManager.setUserLimitFactor(conf.getUserLimitFactor(getQueuePath())); + usersManager.setUserLimit(configuration.getUserLimit(getQueuePath())); + usersManager.setUserLimitFactor(configuration.getUserLimitFactor(getQueuePath())); maxAMResourcePerQueuePercent = - conf.getMaximumApplicationMasterResourcePerQueuePercent( + configuration.getMaximumApplicationMasterResourcePerQueuePercent( getQueuePath()); - maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath()); + maxApplications = configuration.getMaximumApplicationsPerQueue(getQueuePath()); if (maxApplications < 0) { int maxGlobalPerQueueApps = - conf.getGlobalMaximumApplicationsPerQueue(); + configuration.getGlobalMaximumApplicationsPerQueue(); if (maxGlobalPerQueueApps > 0) { maxApplications = maxGlobalPerQueueApps; } } - priorityAcls = conf.getPriorityAcls(getQueuePath(), - originalConfiguration.getClusterLevelApplicationMaxPriority()); + priorityAcls = configuration.getPriorityAcls(getQueuePath(), + configuration.getClusterLevelApplicationMaxPriority()); Set accessibleNodeLabels = this.queueNodeLabelsSettings.getAccessibleNodeLabels(); if (!SchedulerUtils.checkQueueLabelExpression(accessibleNodeLabels, @@ -224,10 +221,10 @@ protected void setupQueueConfigs(Resource clusterResource, .join(getAccessibleNodeLabels().iterator(), ','))); } - nodeLocalityDelay = originalConfiguration.getNodeLocalityDelay(); - rackLocalityAdditionalDelay = originalConfiguration + nodeLocalityDelay = configuration.getNodeLocalityDelay(); + rackLocalityAdditionalDelay = configuration .getRackLocalityAdditionalDelay(); - rackLocalityFullReset = originalConfiguration + rackLocalityFullReset = configuration .getRackLocalityFullReset(); // re-init this since max allocation could have changed @@ -250,10 +247,10 @@ protected void setupQueueConfigs(Resource clusterResource, } defaultAppPriorityPerQueue = Priority.newInstance( - conf.getDefaultApplicationPriorityConfPerQueue(getQueuePath())); + configuration.getDefaultApplicationPriorityConfPerQueue(getQueuePath())); // Validate leaf queue's user's weights. - float queueUserLimit = Math.min(100.0f, conf.getUserLimit(getQueuePath())); + float queueUserLimit = Math.min(100.0f, configuration.getUserLimit(getQueuePath())); getUserWeights().validateForLeafQueue(queueUserLimit, getQueuePath()); usersManager.updateUserWeights(); @@ -529,9 +526,8 @@ public List getPriorityACLs() { } } - protected void reinitialize( - CSQueue newlyParsedQueue, Resource clusterResource, - CapacitySchedulerConfiguration configuration) throws + @Override + public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) throws IOException { writeLock.lock(); @@ -565,20 +561,12 @@ protected void reinitialize( + newMax); } - setupQueueConfigs(clusterResource, configuration); + setupQueueConfigs(clusterResource); } finally { writeLock.unlock(); } } - @Override - public void reinitialize( - CSQueue newlyParsedQueue, Resource clusterResource) - throws IOException { - reinitialize(newlyParsedQueue, clusterResource, - queueContext.getConfiguration()); - } - @Override public void submitApplicationAttempt(FiCaSchedulerApp application, String userName) { @@ -1700,13 +1688,13 @@ protected boolean canAssignToUser(Resource clusterResource, } @Override - protected void setDynamicQueueProperties(CapacitySchedulerConfiguration configuration) { + protected void setDynamicQueueProperties() { // set to -1, to disable it - configuration.setUserLimitFactor(getQueuePath(), -1); + queueContext.getConfiguration().setUserLimitFactor(getQueuePath(), -1); // Set Max AM percentage to a higher value - configuration.setMaximumApplicationMasterResourcePerQueuePercent( + queueContext.getConfiguration().setMaximumApplicationMasterResourcePerQueuePercent( getQueuePath(), 1f); - super.setDynamicQueueProperties(configuration); + super.setDynamicQueueProperties(); } private void updateSchedulerHealthForCompletedContainer( @@ -1948,7 +1936,7 @@ public void updateClusterResource(Resource clusterResource, super.updateEffectiveResources(clusterResource); // Update maximum applications for the queue and for users - updateMaximumApplications(queueContext.getConfiguration()); + updateMaximumApplications(); updateCurrentResourceLimits(currentResourceLimits, clusterResource); @@ -2342,11 +2330,12 @@ public void stopQueue() { } } - void updateMaximumApplications(CapacitySchedulerConfiguration conf) { - int maxAppsForQueue = conf.getMaximumApplicationsPerQueue(getQueuePath()); + void updateMaximumApplications() { + CapacitySchedulerConfiguration configuration = queueContext.getConfiguration(); + int maxAppsForQueue = configuration.getMaximumApplicationsPerQueue(getQueuePath()); - int maxDefaultPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue(); - int maxSystemApps = conf.getMaximumSystemApplications(); + int maxDefaultPerQueueApps = configuration.getGlobalMaximumApplicationsPerQueue(); + int maxSystemApps = configuration.getMaximumSystemApplications(); int baseMaxApplications = maxDefaultPerQueueApps > 0 ? Math.min(maxDefaultPerQueueApps, maxSystemApps) : maxSystemApps; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java index 6d272184100bad..1c25ce2928bf83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java @@ -19,7 +19,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common @@ -55,7 +54,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) writeLock.lock(); try { // Set new configs - setupQueueConfigs(clusterResource, queueContext.getConfiguration()); + setupQueueConfigs(clusterResource); } finally { writeLock.unlock(); @@ -175,22 +174,12 @@ protected CapacitySchedulerConfiguration initializeLeafQueueConfigs(String CapacitySchedulerConfiguration leafQueueConfigs = new CapacitySchedulerConfiguration(new Configuration(false), false); - Map rtProps = queueContext - .getConfiguration().getConfigurationProperties() - .getPropertiesWithPrefix(YarnConfiguration.RESOURCE_TYPES + ".", true); - for (Map.Entry entry : rtProps.entrySet()) { - leafQueueConfigs.set(entry.getKey(), entry.getValue()); - } - Map templateConfigs = queueContext .getConfiguration().getConfigurationProperties() .getPropertiesWithPrefix(configPrefix, true); - for (final Iterator> iterator = - templateConfigs.entrySet().iterator(); iterator.hasNext(); ) { - Map.Entry confKeyValuePair = iterator.next(); - leafQueueConfigs.set(confKeyValuePair.getKey(), - confKeyValuePair.getValue()); + for (Map.Entry confKeyValuePair : templateConfigs.entrySet()) { + leafQueueConfigs.set(confKeyValuePair.getKey(), confKeyValuePair.getValue()); } return leafQueueConfigs; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java index 2d818cd1556a58..384a652e234352 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java @@ -42,7 +42,8 @@ public class AutoCreatedLeafQueue extends AbstractAutoCreatedLeafQueue { public AutoCreatedLeafQueue(CapacitySchedulerQueueContext queueContext, String queueName, ManagedParentQueue parent) throws IOException { super(queueContext, queueName, parent, null); - super.setupQueueConfigs(queueContext.getClusterResource(), parent.getLeafQueueConfigs(queueName)); + parent.setLeafQueueConfigs(queueName); + super.setupQueueConfigs(queueContext.getClusterResource()); updateCapacitiesToZero(); } @@ -56,8 +57,8 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) ManagedParentQueue managedParentQueue = (ManagedParentQueue) parent; - super.reinitialize(newlyParsedQueue, clusterResource, managedParentQueue - .getLeafQueueConfigs(newlyParsedQueue.getQueueShortName())); + managedParentQueue.setLeafQueueConfigs(newlyParsedQueue.getQueueShortName()); + super.reinitialize(newlyParsedQueue, clusterResource); //Reset capacities to 0 since reinitialize above // queueCapacities to initialize to configured capacity which might @@ -122,8 +123,7 @@ public void validateConfigurations(AutoCreatedLeafQueueConfig template) } @Override - protected void setDynamicQueueProperties( - CapacitySchedulerConfiguration configuration) { + protected void setDynamicQueueProperties() { String parentTemplate = String.format("%s.%s", getParent().getQueuePath(), CapacitySchedulerConfiguration .AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java index 56874888870b86..dc254747dfec50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java @@ -26,13 +26,10 @@ public class CSQueuePreemptionSettings { public CSQueuePreemptionSettings( CSQueue queue, - CapacitySchedulerConfiguration configuration, - CapacitySchedulerConfiguration originalSchedulerConfiguration) { - this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(queue, configuration, - originalSchedulerConfiguration); + CapacitySchedulerConfiguration configuration) { + this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(queue, configuration); this.intraQueuePreemptionDisabledInHierarchy = - isIntraQueueHierarchyPreemptionDisabled(queue, configuration, - originalSchedulerConfiguration); + isIntraQueueHierarchyPreemptionDisabled(queue, configuration); } /** @@ -46,10 +43,9 @@ public CSQueuePreemptionSettings( * @return true if queue has cross-queue preemption disabled, false otherwise */ private boolean isQueueHierarchyPreemptionDisabled(CSQueue q, - CapacitySchedulerConfiguration configuration, - CapacitySchedulerConfiguration originalSchedulerConfiguration) { + CapacitySchedulerConfiguration configuration) { boolean systemWidePreemption = - originalSchedulerConfiguration + configuration .getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS); CSQueue parentQ = q.getParent(); @@ -85,10 +81,9 @@ private boolean isQueueHierarchyPreemptionDisabled(CSQueue q, * @return true if queue has intra-queue preemption disabled, false otherwise */ private boolean isIntraQueueHierarchyPreemptionDisabled(CSQueue q, - CapacitySchedulerConfiguration configuration, - CapacitySchedulerConfiguration originalSchedulerConfiguration) { + CapacitySchedulerConfiguration configuration) { boolean systemWideIntraQueuePreemption = - originalSchedulerConfiguration.getBoolean( + configuration.getBoolean( CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ENABLED, CapacitySchedulerConfiguration .DEFAULT_INTRAQUEUE_PREEMPTION_ENABLED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java index e9ec3a0e490649..df7a6274566f5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java @@ -96,6 +96,10 @@ public CapacitySchedulerConfiguration getConfiguration() { return configuration; } + public void setConfigurationEntry(String name, String value) { + this.configuration.set(name, value); + } + public Resource getMinimumAllocation() { return minimumAllocation; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index ee53c14f8b0b00..f33de96e2c8814 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -41,6 +41,6 @@ public LeafQueue(CapacitySchedulerQueueContext queueContext, IOException { super(queueContext, queueName, parent, old, isDynamic); - setupQueueConfigs(queueContext.getClusterResource(), queueContext.getConfiguration()); + setupQueueConfigs(queueContext.getClusterResource()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index 0aab2e412f3966..7b019d90667de0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; @@ -33,7 +32,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -454,25 +452,13 @@ private void applyQueueManagementChanges( } } - public CapacitySchedulerConfiguration getLeafQueueConfigs( - String leafQueueName) { - return getLeafQueueConfigs(getLeafQueueTemplate().getLeafQueueConfigs(), - leafQueueName); - } - - public CapacitySchedulerConfiguration getLeafQueueConfigs( - CapacitySchedulerConfiguration templateConfig, String leafQueueName) { - CapacitySchedulerConfiguration leafQueueConfigTemplate = new - CapacitySchedulerConfiguration(new Configuration(false), false); - for (final Iterator> iterator = - templateConfig.iterator(); iterator.hasNext();) { - Map.Entry confKeyValuePair = iterator.next(); - final String name = confKeyValuePair.getKey().replaceFirst( - CapacitySchedulerConfiguration - .AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX, - leafQueueName); - leafQueueConfigTemplate.set(name, confKeyValuePair.getValue()); + public void setLeafQueueConfigs(String leafQueueName) { + CapacitySchedulerConfiguration templateConfig = leafQueueTemplate.getLeafQueueConfigs(); + for (Map.Entry confKeyValuePair : templateConfig) { + final String name = confKeyValuePair.getKey() + .replaceFirst(CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX, + leafQueueName); + queueContext.setConfigurationEntry(name, confKeyValuePair.getValue()); } - return leafQueueConfigTemplate; } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index b2ff8995ff3cfd..283d5678b8c91f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -134,7 +134,7 @@ private ParentQueue(CapacitySchedulerQueueContext queueContext, queueContext.getConfiguration() .getAllowZeroCapacitySum(getQueuePath()); - setupQueueConfigs(queueContext.getClusterResource(), queueContext.getConfiguration()); + setupQueueConfigs(queueContext.getClusterResource()); } // returns what is configured queue ordering policy @@ -144,14 +144,14 @@ private String getQueueOrderingPolicyConfigName() { queueOrderingPolicy.getConfigName(); } - protected void setupQueueConfigs(Resource clusterResource, - CapacitySchedulerConfiguration configuration) + protected void setupQueueConfigs(Resource clusterResource) throws IOException { writeLock.lock(); try { + CapacitySchedulerConfiguration configuration = queueContext.getConfiguration(); autoCreatedQueueTemplate = new AutoCreatedQueueTemplate( configuration, this.queuePath); - super.setupQueueConfigs(clusterResource, configuration); + super.setupQueueConfigs(clusterResource); StringBuilder aclsString = new StringBuilder(); for (Map.Entry e : getACLs().entrySet()) { aclsString.append(e.getKey()).append(":") @@ -633,7 +633,7 @@ public void reinitialize(CSQueue newlyParsedQueue, ParentQueue newlyParsedParentQueue = (ParentQueue) newlyParsedQueue; // Set new configs - setupQueueConfigs(clusterResource, queueContext.getConfiguration()); + setupQueueConfigs(clusterResource); // Re-configure existing child queues and add new ones // The CS has already checked to ensure all existing child queues are present! diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java index 2b182e532f4a92..cca46f50095d7b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java @@ -104,7 +104,7 @@ public void reinitialize(CSQueue newlyParsedQueue, } // Set new configs - setupQueueConfigs(clusterResource, queueContext.getConfiguration()); + setupQueueConfigs(clusterResource); updateQuotas(newlyParsedParentQueue.userLimit, newlyParsedParentQueue.userLimitFactor, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java index 730b797104fe3a..101c8076fdc04f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java @@ -36,17 +36,10 @@ public QueueAllocationSettings(Resource minimumAllocation) { this.minimumAllocation = minimumAllocation; } - void setupMaximumAllocation(CapacitySchedulerConfiguration configuration, - CapacitySchedulerConfiguration originalSchedulerConfiguration, String queuePath, + void setupMaximumAllocation(CapacitySchedulerConfiguration configuration, String queuePath, CSQueue parent) { - /* YARN-10869: When using AutoCreatedLeafQueues, the passed configuration - * object is a cloned one containing only the template configs - * (see ManagedParentQueue#getLeafQueueConfigs). To ensure that the actual - * cluster maximum allocation is fetched the original config object should - * be used. - */ Resource clusterMax = ResourceUtils - .fetchMaximumAllocationFromConfig(originalSchedulerConfiguration); + .fetchMaximumAllocationFromConfig(configuration); Resource queueMax = configuration.getQueueMaximumAllocation(queuePath); maximumAllocation = Resources.clone( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java index 38ee4d237a6f0a..7b3144b6a8b350 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java @@ -39,8 +39,7 @@ public class ReservationQueue extends AbstractAutoCreatedLeafQueue { public ReservationQueue(CapacitySchedulerQueueContext queueContext, String queueName, PlanQueue parent) throws IOException { super(queueContext, queueName, parent, null); - super.setupQueueConfigs(queueContext.getClusterResource(), - queueContext.getConfiguration()); + super.setupQueueConfigs(queueContext.getClusterResource()); // the following parameters are common to all reservation in the plan updateQuotas(parent.getUserLimitForReservation(), @@ -84,8 +83,7 @@ private void updateQuotas(float userLimit, float userLimitFactor, } @Override - protected void setupConfigurableCapacities(CapacitySchedulerConfiguration - configuration) { + protected void setupConfigurableCapacities() { super.updateAbsoluteCapacities(); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java index 6c84c8eab5ef15..b71fe063927ac8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java @@ -177,7 +177,7 @@ private void testUserLimitThroughputWithNumberOfResourceTypes( LeafQueue qb = (LeafQueue)cs.getQueue(queueName); // For now make user limit large so we can activate all applications qb.setUserLimitFactor((float)100.0); - qb.setupConfigurableCapacities(cs.getConfiguration()); + qb.setupConfigurableCapacities(); lqs[i] = qb; } From b504becced18fa61d93fa41c6da68dbd7e1a68f4 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Tue, 14 Dec 2021 22:00:43 +0100 Subject: [PATCH 15/33] Clean up checkstyle warnings from YARN-11024/10907/10929. Contributed by Benjamin Teke --- .../scheduler/capacity/AbstractCSQueue.java | 3 +- .../scheduler/capacity/AbstractLeafQueue.java | 38 ++++++++----------- ...uaranteedOrZeroCapacityOverTimePolicy.java | 1 - .../capacity/TestApplicationLimits.java | 35 ++++++++--------- .../TestApplicationLimitsByPartition.java | 1 - .../scheduler/capacity/TestReservations.java | 1 - 6 files changed, 33 insertions(+), 46 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 3a0e2ae4d838c2..809a8603e25f04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -145,7 +145,8 @@ public AbstractCSQueue(CapacitySchedulerQueueContext queueContext, String queueN CSQueueMetrics metrics = old != null ? (CSQueueMetrics) old.getMetrics() : CSQueueMetrics.forQueue(getQueuePath(), parent, - queueContext.getConfiguration().getEnableUserMetrics(), queueContext.getConfiguration()); + queueContext.getConfiguration().getEnableUserMetrics(), + queueContext.getConfiguration()); this.usageTracker = new CSQueueUsageTracker(metrics); this.queueCapacities = new QueueCapacities(parent == null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java index 8b31241d527151..e194800cd1b86d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java @@ -453,8 +453,7 @@ public QueueInfo getQueueInfo( } @Override - public List - getQueueUserAclInfo(UserGroupInformation user) { + public List getQueueUserAclInfo(UserGroupInformation user) { readLock.lock(); try { QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance( @@ -527,8 +526,8 @@ public List getPriorityACLs() { } @Override - public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) throws - IOException { + public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) + throws IOException { writeLock.lock(); try { @@ -634,7 +633,8 @@ public void validateSubmitApplication(ApplicationId applicationId, // Check submission limits for queues //TODO recalculate max applications because they can depend on capacity - if (getNumApplications() >= getMaxApplications() && !(this instanceof AutoCreatedLeafQueue)) { + if (getNumApplications() >= getMaxApplications() && + !(this instanceof AutoCreatedLeafQueue)) { String msg = "Queue " + getQueuePath() + " already has " + getNumApplications() + " applications," @@ -646,7 +646,8 @@ public void validateSubmitApplication(ApplicationId applicationId, // Check submission limits for the user on this queue User user = usersManager.getUserAndAddIfAbsent(userName); //TODO recalculate max applications because they can depend on capacity - if (user.getTotalApplications() >= getMaxApplicationsPerUser() && !(this instanceof AutoCreatedLeafQueue)) { + if (user.getTotalApplications() >= getMaxApplicationsPerUser() && + !(this instanceof AutoCreatedLeafQueue)) { String msg = "Queue " + getQueuePath() + " already has " + user .getTotalApplications() + " applications from user " + userName + " cannot accept submission of application: " + applicationId; @@ -825,10 +826,9 @@ protected void activateApplications() { calculateAndGetAMResourceLimitPerPartition(nodePartition); } - for (Iterator fsApp = - getPendingAppsOrderingPolicy() + for (Iterator fsApp = getPendingAppsOrderingPolicy() .getAssignmentIterator(IteratorSelector.EMPTY_ITERATOR_SELECTOR); - fsApp.hasNext(); ) { + fsApp.hasNext();) { FiCaSchedulerApp application = fsApp.next(); ApplicationId applicationId = application.getApplicationId(); @@ -864,7 +864,8 @@ protected void activateApplications() { + " skipping enforcement to allow at least one application" + " to start"); } else{ - application.updateAMContainerDiagnostics(SchedulerApplicationAttempt.AMState.INACTIVATED, + application.updateAMContainerDiagnostics( + SchedulerApplicationAttempt.AMState.INACTIVATED, CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED); LOG.debug("Not activating application {} as amIfStarted: {}" + " exceeds amLimit: {}", applicationId, amIfStarted, amLimit); @@ -1189,9 +1190,8 @@ public CSAssignment assignContainers(Resource clusterResource, boolean needAssignToQueueCheck = true; IteratorSelector sel = new IteratorSelector(); sel.setPartition(candidates.getPartition()); - for (Iterator assignmentIterator = - orderingPolicy.getAssignmentIterator(sel); - assignmentIterator.hasNext(); ) { + for (Iterator assignmentIterator = orderingPolicy.getAssignmentIterator(sel); + assignmentIterator.hasNext();) { FiCaSchedulerApp application = assignmentIterator.next(); ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager, @@ -1821,13 +1821,8 @@ void allocateResource(Resource clusterResource, if (null != rmContainer && rmContainer.getNodeLabelExpression().equals( RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals( RMNodeLabelsManager.NO_LABEL)) { - TreeSet rmContainers = null; - if (null == (rmContainers = ignorePartitionExclusivityRMContainers.get( - nodePartition))) { - rmContainers = new TreeSet<>(); - ignorePartitionExclusivityRMContainers.put(nodePartition, - rmContainers); - } + TreeSet rmContainers = ignorePartitionExclusivityRMContainers.computeIfAbsent( + nodePartition, k -> new TreeSet<>()); rmContainers.add(rmContainer); } @@ -2195,8 +2190,7 @@ public void detachContainer(Resource clusterResource, * @return all ignored partition exclusivity RMContainers in the LeafQueue, * this will be used by preemption policy. */ - public Map> - getIgnoreExclusivityRMContainers() { + public Map> getIgnoreExclusivityRMContainers() { Map> clonedMap = new HashMap<>(); readLock.lock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java index 14d3555e100a87..35275574b74fb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java @@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractAutoCreatedLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueueConfig; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 33134babc9f57d..7cb0ccd3049b0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -275,8 +275,9 @@ public void testLimitsComputation() throws Exception { Resource clusterResource = Resources.createResource(100 * 16 * GB, 100 * 16); - CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator, Resources.createResource(GB, 1), - Resources.createResource(16*GB, 16), clusterResource); + CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator, + Resources.createResource(GB, 1), Resources.createResource(16*GB, 16), + clusterResource); CapacitySchedulerQueueManager queueManager = csContext.getCapacitySchedulerQueueManager(); CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext); @@ -299,7 +300,7 @@ public void testLimitsComputation() throws Exception { assertThat(queue.calculateAndGetAMResourceLimit()). isEqualTo(amResourceLimit); assertThat(queue.getUserAMResourceLimit()).isEqualTo( - Resource.newInstance(80*GB, 1)); + Resource.newInstance(80*GB, 1)); // Assert in metrics assertThat(queue.getMetrics().getAMResourceLimitMB()).isEqualTo( @@ -307,10 +308,8 @@ public void testLimitsComputation() throws Exception { assertThat(queue.getMetrics().getAMResourceLimitVCores()).isEqualTo( amResourceLimit.getVirtualCores()); - assertEquals( - (int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()), - queue.getMetrics().getAvailableMB() - ); + assertEquals((int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()), + queue.getMetrics().getAvailableMB()); // Add some nodes to the cluster & test new limits clusterResource = Resources.createResource(120 * 16 * GB); @@ -322,10 +321,8 @@ public void testLimitsComputation() throws Exception { assertThat(queue.getUserAMResourceLimit()).isEqualTo( Resource.newInstance(96*GB, 1)); - assertEquals( - (int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()), - queue.getMetrics().getAvailableMB() - ); + assertEquals((int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()), + queue.getMetrics().getAvailableMB()); // should return -1 if per queue setting not set assertEquals( @@ -343,11 +340,10 @@ public void testLimitsComputation() throws Exception { assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser()); // should default to global setting if per queue setting not set - assertEquals( - (long)CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, + assertEquals((long) + CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, (long)csConf.getMaximumApplicationMasterResourcePerQueuePercent( - queue.getQueuePath()) - ); + queue.getQueuePath())); // Change the per-queue max AM resources percentage. csConf.setFloat(PREFIX + queue.getQueuePath() @@ -365,10 +361,9 @@ public void testLimitsComputation() throws Exception { queue = (LeafQueue)queues.get(A); - assertEquals((long) 0.5, + assertEquals((long) 0.5, (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent( - queue.getQueuePath()) - ); + queue.getQueuePath())); assertThat(queue.calculateAndGetAMResourceLimit()).isEqualTo( Resource.newInstance(800 * GB, 1)); @@ -579,8 +574,8 @@ public void testHeadroom() throws Exception { // Say cluster has 100 nodes of 16G each Resource clusterResource = Resources.createResource(100 * 16 * GB); - CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator, Resources.createResource(GB), - Resources.createResource(16*GB), clusterResource); + CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator, + Resources.createResource(GB), Resources.createResource(16*GB), clusterResource); CapacitySchedulerQueueManager queueManager = csContext.getCapacitySchedulerQueueManager(); CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index 4c2ec87e705299..ef50e5271e7188 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index 5662df4c510371..53b1d160dc2898 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; From 84da38b4a89c260a690512d029dcb9f3ada00589 Mon Sep 17 00:00:00 2001 From: litao Date: Wed, 15 Dec 2021 11:16:32 +0800 Subject: [PATCH 16/33] HDFS-16378. Add datanode address to BlockReportLeaseManager logs (#3786). Contributed by tomscut. Signed-off-by: He Xiaoqiao --- .../BlockReportLeaseManager.java | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java index f45daac142c86c..2e7e78d14e2c94 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java @@ -190,8 +190,8 @@ public synchronized void register(DatanodeDescriptor dn) { private synchronized NodeData registerNode(DatanodeDescriptor dn) { if (nodes.containsKey(dn.getDatanodeUuid())) { - LOG.info("Can't register DN {} because it is already registered.", - dn.getDatanodeUuid()); + LOG.info("Can't register DN {} ({}) because it is already registered.", + dn.getDatanodeUuid(), dn.getXferAddr()); return null; } NodeData node = new NodeData(dn.getDatanodeUuid()); @@ -213,8 +213,8 @@ private synchronized void remove(NodeData node) { public synchronized void unregister(DatanodeDescriptor dn) { NodeData node = nodes.remove(dn.getDatanodeUuid()); if (node == null) { - LOG.info("Can't unregister DN {} because it is not currently " + - "registered.", dn.getDatanodeUuid()); + LOG.info("Can't unregister DN {} ({}) because it is not currently " + + "registered.", dn.getDatanodeUuid(), dn.getXferAddr()); return; } remove(node); @@ -224,7 +224,7 @@ public synchronized long requestLease(DatanodeDescriptor dn) { NodeData node = nodes.get(dn.getDatanodeUuid()); if (node == null) { LOG.warn("DN {} ({}) requested a lease even though it wasn't yet " + - "registered. Registering now.", dn.getDatanodeUuid(), + "registered. Registering now.", dn.getDatanodeUuid(), dn.getXferAddr()); node = registerNode(dn); } @@ -232,9 +232,9 @@ public synchronized long requestLease(DatanodeDescriptor dn) { // The DataNode wants a new lease, even though it already has one. // This can happen if the DataNode is restarted in between requesting // a lease and using it. - LOG.debug("Removing existing BR lease 0x{} for DN {} in order to " + + LOG.debug("Removing existing BR lease 0x{} for DN {} ({}) in order to " + "issue a new one.", Long.toHexString(node.leaseId), - dn.getDatanodeUuid()); + dn.getDatanodeUuid(), dn.getXferAddr()); } remove(node); long monotonicNowMs = Time.monotonicNow(); @@ -248,9 +248,9 @@ public synchronized long requestLease(DatanodeDescriptor dn) { allLeases.append(prefix).append(cur.datanodeUuid); prefix = ", "; } - LOG.debug("Can't create a new BR lease for DN {}, because " + - "numPending equals maxPending at {}. Current leases: {}", - dn.getDatanodeUuid(), numPending, allLeases.toString()); + LOG.debug("Can't create a new BR lease for DN {} ({}), because " + + "numPending equals maxPending at {}. Current leases: {}", + dn.getDatanodeUuid(), dn.getXferAddr(), numPending, allLeases); } return 0; } @@ -259,8 +259,8 @@ public synchronized long requestLease(DatanodeDescriptor dn) { node.leaseTimeMs = monotonicNowMs; pendingHead.addToEnd(node); if (LOG.isDebugEnabled()) { - LOG.debug("Created a new BR lease 0x{} for DN {}. numPending = {}", - Long.toHexString(node.leaseId), dn.getDatanodeUuid(), numPending); + LOG.debug("Created a new BR lease 0x{} for DN {} ({}). numPending = {}", + Long.toHexString(node.leaseId), dn.getDatanodeUuid(), dn.getXferAddr(), numPending); } return node.leaseId; } @@ -293,36 +293,36 @@ private synchronized void pruneExpiredPending(long monotonicNowMs) { public synchronized boolean checkLease(DatanodeDescriptor dn, long monotonicNowMs, long id) { if (id == 0) { - LOG.debug("Datanode {} is using BR lease id 0x0 to bypass " + - "rate-limiting.", dn.getDatanodeUuid()); + LOG.debug("Datanode {} ({}) is using BR lease id 0x0 to bypass " + + "rate-limiting.", dn.getDatanodeUuid(), dn.getXferAddr()); return true; } NodeData node = nodes.get(dn.getDatanodeUuid()); if (node == null) { - LOG.info("BR lease 0x{} is not valid for unknown datanode {}", - Long.toHexString(id), dn.getDatanodeUuid()); + LOG.info("BR lease 0x{} is not valid for unknown datanode {} ({})", + Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr()); return false; } if (node.leaseId == 0) { - LOG.warn("BR lease 0x{} is not valid for DN {}, because the DN " + + LOG.warn("BR lease 0x{} is not valid for DN {} ({}), because the DN " + "is not in the pending set.", - Long.toHexString(id), dn.getDatanodeUuid()); + Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr()); return false; } if (pruneIfExpired(monotonicNowMs, node)) { - LOG.warn("BR lease 0x{} is not valid for DN {}, because the lease " + - "has expired.", Long.toHexString(id), dn.getDatanodeUuid()); + LOG.warn("BR lease 0x{} is not valid for DN {} ({}), because the lease " + + "has expired.", Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr()); return false; } if (id != node.leaseId) { - LOG.warn("BR lease 0x{} is not valid for DN {}. Expected BR lease 0x{}.", - Long.toHexString(id), dn.getDatanodeUuid(), + LOG.warn("BR lease 0x{} is not valid for DN {} ({}). Expected BR lease 0x{}.", + Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr(), Long.toHexString(node.leaseId)); return false; } if (LOG.isTraceEnabled()) { - LOG.trace("BR lease 0x{} is valid for DN {}.", - Long.toHexString(id), dn.getDatanodeUuid()); + LOG.trace("BR lease 0x{} is valid for DN {} ({}).", + Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr()); } return true; } @@ -330,20 +330,20 @@ public synchronized boolean checkLease(DatanodeDescriptor dn, public synchronized long removeLease(DatanodeDescriptor dn) { NodeData node = nodes.get(dn.getDatanodeUuid()); if (node == null) { - LOG.info("Can't remove lease for unknown datanode {}", - dn.getDatanodeUuid()); + LOG.info("Can't remove lease for unknown datanode {} ({})", + dn.getDatanodeUuid(), dn.getXferAddr()); return 0; } long id = node.leaseId; if (id == 0) { - LOG.debug("DN {} has no lease to remove.", dn.getDatanodeUuid()); + LOG.debug("DN {} ({}) has no lease to remove.", dn.getDatanodeUuid(), dn.getXferAddr()); return 0; } remove(node); deferredHead.addToEnd(node); if (LOG.isTraceEnabled()) { - LOG.trace("Removed BR lease 0x{} for DN {}. numPending = {}", - Long.toHexString(id), dn.getDatanodeUuid(), numPending); + LOG.trace("Removed BR lease 0x{} for DN {} ({}). numPending = {}", + Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr(), numPending); } return id; } From 2d1142ade3e6730cfce7cd2969e822c87f40a109 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Wed, 15 Dec 2021 14:17:51 +0530 Subject: [PATCH 17/33] YARN-11045. ATSv2 storage monitor fails to read from hbase cluster (#3796) --- hadoop-yarn-project/hadoop-yarn/bin/yarn | 2 +- hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 2 +- .../pom.xml | 42 +++++++++++++++++++ .../pom.xml | 22 ++++++++++ .../pom.xml | 32 +++++++++++++- .../pom.xml | 32 +++++++++++++- 6 files changed, 128 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index dbab397f26c185..5eccaadeb605b5 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -193,7 +193,7 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" timelinereader) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*" - hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" before HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' ;; nodeattributes) diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 3a60794fad1657..4508ad38b66f0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -251,7 +251,7 @@ goto :eof :timelinereader set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\timelineserver-config\log4j.properties set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\* - set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\* + set CLASSPATH=%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\*;%CLASSPATH% set CLASS=org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer set YARN_OPTS=%YARN_OPTS% %YARN_TIMELINEREADER_OPTS% goto :eof diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml index 9a88e831095266..f6ea8660e83112 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml @@ -37,6 +37,12 @@ org.apache.hadoop hadoop-yarn-server-timelineservice-hbase-common + + + com.google.guava + guava + + @@ -54,6 +60,12 @@ hadoop-shaded-guava + + com.google.guava + guava + ${hbase-compatible-guava.version} + + org.apache.hadoop hadoop-annotations @@ -64,6 +76,12 @@ org.apache.hadoop hadoop-common provided + + + com.google.guava + guava + + @@ -72,6 +90,12 @@ hadoop-common test-jar test + + + com.google.guava + guava + + @@ -108,6 +132,12 @@ org.apache.hadoop hadoop-yarn-server-timelineservice + + + com.google.guava + guava + + @@ -128,6 +158,10 @@ org.mortbay.jetty jetty-util + + com.google.guava + guava + @@ -139,6 +173,10 @@ org.apache.hadoop hadoop-mapreduce-client-core + + com.google.guava + guava + @@ -187,6 +225,10 @@ org.eclipse.jetty jetty-http + + com.google.guava + guava + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml index b1ff0ca437c9f9..247981d61d6722 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml @@ -51,6 +51,12 @@ org.apache.hadoop hadoop-yarn-server-timelineservice + + + com.google.guava + guava + + @@ -67,6 +73,12 @@ hadoop-common test-jar test + + + com.google.guava + guava + + @@ -81,6 +93,10 @@ org.mortbay.jetty jetty-util + + com.google.guava + guava + @@ -90,6 +106,12 @@ test + + com.google.guava + guava + ${hbase-compatible-guava.version} + + junit junit diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml index 68d6d960e56c48..d7571a456e2097 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml @@ -27,7 +27,7 @@ 4.0.0 hadoop-yarn-server-timelineservice-hbase-server-1 - Apache Hadoop YARN TimelineService HBase Server 1.2 + Apache Hadoop YARN TimelineService HBase Server 1.7 3.4.0-SNAPSHOT @@ -47,6 +47,12 @@ org.apache.hadoop hadoop-yarn-server-timelineservice-hbase-common + + + com.google.guava + guava + + @@ -59,6 +65,12 @@ hadoop-shaded-guava + + com.google.guava + guava + ${hbase-compatible-guava.version} + + org.apache.hadoop hadoop-annotations @@ -69,6 +81,12 @@ org.apache.hadoop hadoop-common provided + + + com.google.guava + guava + + @@ -89,6 +107,10 @@ org.mortbay.jetty jetty-util + + com.google.guava + guava + @@ -100,6 +122,10 @@ org.apache.hadoop hadoop-mapreduce-client-core + + com.google.guava + guava + @@ -136,6 +162,10 @@ org.mortbay.jetty jetty-sslengine + + com.google.guava + guava + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml index df01ad7549c123..b19188eefe425a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml @@ -27,7 +27,7 @@ 4.0.0 hadoop-yarn-server-timelineservice-hbase-server-2 - Apache Hadoop YARN TimelineService HBase Server 2.0 + Apache Hadoop YARN TimelineService HBase Server 2.2 3.4.0-SNAPSHOT @@ -47,6 +47,12 @@ org.apache.hadoop hadoop-yarn-server-timelineservice-hbase-common + + + com.google.guava + guava + + @@ -59,6 +65,12 @@ hadoop-shaded-guava + + com.google.guava + guava + ${hbase-compatible-guava.version} + + org.apache.hadoop hadoop-annotations @@ -69,6 +81,12 @@ org.apache.hadoop hadoop-common provided + + + com.google.guava + guava + + @@ -89,6 +107,10 @@ org.mortbay.jetty jetty-util + + com.google.guava + guava + @@ -100,6 +122,10 @@ org.apache.hadoop hadoop-mapreduce-client-core + + com.google.guava + guava + @@ -155,6 +181,10 @@ org.eclipse.jetty jetty-http + + com.google.guava + guava + From 8a1e4cfd628d7a0719846b5de2595bf2178c2f41 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Thu, 16 Dec 2021 00:01:09 +0100 Subject: [PATCH 18/33] YARN-11044. Fix TestApplicationLimits.testLimitsComputation() ineffective asserts. Contributed by Benjamin Teke --- .../scheduler/capacity/TestApplicationLimits.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 7cb0ccd3049b0d..f06631d6c15d5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -267,6 +267,8 @@ public void testAMResourceLimit() throws Exception { @Test public void testLimitsComputation() throws Exception { + final float epsilon = 1e-5f; + CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(csConf); @@ -340,10 +342,9 @@ public void testLimitsComputation() throws Exception { assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser()); // should default to global setting if per queue setting not set - assertEquals((long) - CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, - (long)csConf.getMaximumApplicationMasterResourcePerQueuePercent( - queue.getQueuePath())); + assertEquals(CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, + csConf.getMaximumApplicationMasterResourcePerQueuePercent( + queue.getQueuePath()), epsilon); // Change the per-queue max AM resources percentage. csConf.setFloat(PREFIX + queue.getQueuePath() @@ -361,9 +362,9 @@ public void testLimitsComputation() throws Exception { queue = (LeafQueue)queues.get(A); - assertEquals((long) 0.5, - (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent( - queue.getQueuePath())); + assertEquals(0.5f, + csConf.getMaximumApplicationMasterResourcePerQueuePercent( + queue.getQueuePath()), epsilon); assertThat(queue.calculateAndGetAMResourceLimit()).isEqualTo( Resource.newInstance(800 * GB, 1)); From 2d2345aab46e17f3b8b04f3afcfe37c2bdda47f9 Mon Sep 17 00:00:00 2001 From: litao Date: Thu, 16 Dec 2021 12:29:32 +0800 Subject: [PATCH 19/33] HDFS-16375. The FBR lease ID should be exposed to the log (#3769) --- .../server/blockmanagement/BlockManager.java | 18 ++++++++++-------- .../hdfs/server/datanode/BPServiceActor.java | 4 ++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 9ec9f9bd472248..3348afa6e738da 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2741,6 +2741,8 @@ public boolean processReport(final DatanodeID nodeID, Collection invalidatedBlocks = Collections.emptyList(); String strBlockReportId = context != null ? Long.toHexString(context.getReportId()) : ""; + String fullBrLeaseId = + context != null ? Long.toHexString(context.getLeaseId()) : ""; try { node = datanodeManager.getDatanode(nodeID); @@ -2763,10 +2765,10 @@ public boolean processReport(final DatanodeID nodeID, if (namesystem.isInStartupSafeMode() && !StorageType.PROVIDED.equals(storageInfo.getStorageType()) && storageInfo.getBlockReportCount() > 0) { - blockLog.info("BLOCK* processReport 0x{}: " + blockLog.info("BLOCK* processReport 0x{} with lease ID 0x{}: " + "discarded non-initial block report from {}" + " because namenode still in startup phase", - strBlockReportId, nodeID); + strBlockReportId, fullBrLeaseId, nodeID); blockReportLeaseManager.removeLease(node); return !node.hasStaleStorages(); } @@ -2774,9 +2776,9 @@ public boolean processReport(final DatanodeID nodeID, if (storageInfo.getBlockReportCount() == 0) { // The first block report can be processed a lot more efficiently than // ordinary block reports. This shortens restart times. - blockLog.info("BLOCK* processReport 0x{}: Processing first " + blockLog.info("BLOCK* processReport 0x{} with lease ID 0x{}: Processing first " + "storage report for {} from datanode {}", - strBlockReportId, + strBlockReportId, fullBrLeaseId, storageInfo.getStorageID(), nodeID); processFirstBlockReport(storageInfo, newReport); @@ -2795,8 +2797,8 @@ public boolean processReport(final DatanodeID nodeID, if(blockLog.isDebugEnabled()) { for (Block b : invalidatedBlocks) { - blockLog.debug("BLOCK* processReport 0x{}: {} on node {} size {} " + - "does not belong to any file.", strBlockReportId, b, + blockLog.debug("BLOCK* processReport 0x{} with lease ID 0x{}: {} on node {} size {} " + + "does not belong to any file.", strBlockReportId, fullBrLeaseId, b, node, b.getNumBytes()); } } @@ -2806,9 +2808,9 @@ public boolean processReport(final DatanodeID nodeID, if (metrics != null) { metrics.addStorageBlockReport((int) (endTime - startTime)); } - blockLog.info("BLOCK* processReport 0x{}: from storage {} node {}, " + + blockLog.info("BLOCK* processReport 0x{} with lease ID 0x{}: from storage {} node {}, " + "blocks: {}, hasStaleStorage: {}, processing time: {} msecs, " + - "invalidatedBlocks: {}", strBlockReportId, storage.getStorageID(), + "invalidatedBlocks: {}", strBlockReportId, fullBrLeaseId, storage.getStorageID(), nodeID, newReport.getNumberOfBlocks(), node.hasStaleStorages(), (endTime - startTime), invalidatedBlocks.size()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 1f3147acd9d67b..fe83700d6f9941 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -455,8 +455,8 @@ List blockReport(long fullBrLeaseId) throws IOException { dn.getMetrics().addBlockReport(brSendCost, getRpcMetricSuffix()); final int nCmds = cmds.size(); LOG.info((success ? "S" : "Uns") + - "uccessfully sent block report 0x" + - Long.toHexString(reportId) + " to namenode: " + nnAddr + + "uccessfully sent block report 0x" + Long.toHexString(reportId) + + " with lease ID 0x" + Long.toHexString(fullBrLeaseId) + " to namenode: " + nnAddr + ", containing " + reports.length + " storage report(s), of which we sent " + numReportsSent + "." + " The reports had " + totalBlockCount + From 19e0c0cb45f1171d4daa3e2a7e7a3b0c80f6eb6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20P=C3=A9nzes?= Date: Thu, 16 Dec 2021 05:38:42 +0100 Subject: [PATCH 20/33] HDFS-16384. Upgrade Netty to 4.1.72.Final (#3798) --- LICENSE-binary | 2 +- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 8 ++++++++ hadoop-project/pom.xml | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 6dabe3d71de040..40207ce0e8a634 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -261,7 +261,7 @@ io.grpc:grpc-protobuf:1.26.0 io.grpc:grpc-protobuf-lite:1.26.0 io.grpc:grpc-stub:1.26.0 io.netty:netty:3.10.6.Final -io.netty:netty-all:4.1.42.Final +io.netty:netty-all:4.1.72.Final io.netty:netty-buffer:4.1.27.Final io.netty:netty-codec:4.1.27.Final io.netty:netty-codec-http:4.1.27.Final diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index d65e6030369b3e..bb4bda5ecc70ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -85,6 +85,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> io.netty netty-codec-http + + io.netty + netty-codec-socks + io.netty netty-common @@ -93,6 +97,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> io.netty netty-handler + + io.netty + netty-handler-proxy + io.netty netty-transport diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index cc45975a3f0771..e708402617975d 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -141,7 +141,7 @@ 2.2.4 3.2.4 3.10.6.Final - 4.1.68.Final + 4.1.72.Final 1.1.8.2 1.7.1 From b107c21ce5969e07c556823a2628479cf5697072 Mon Sep 17 00:00:00 2001 From: litao Date: Thu, 16 Dec 2021 12:49:50 +0800 Subject: [PATCH 21/33] HDFS-16377. Should CheckNotNull before access FsDatasetSpi (#3784) Reviewed-by: Viraj Jasani Signed-off-by: Takanobu Asanuma --- .../org/apache/hadoop/hdfs/server/datanode/DataNode.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index a2f00ce53d49c6..c1b00168fc24dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -865,6 +865,7 @@ private void refreshVolumes(String newVolumes) throws IOException { .newFixedThreadPool(changedVolumes.newLocations.size()); List> exceptions = Lists.newArrayList(); + Preconditions.checkNotNull(data, "Storage not yet initialized"); for (final StorageLocation location : changedVolumes.newLocations) { exceptions.add(service.submit(new Callable() { @Override @@ -964,6 +965,7 @@ private synchronized void removeVolumes( clearFailure, Joiner.on(",").join(storageLocations))); IOException ioe = null; + Preconditions.checkNotNull(data, "Storage not yet initialized"); // Remove volumes and block infos from FsDataset. data.removeVolumes(storageLocations, clearFailure); @@ -2040,6 +2042,7 @@ FileInputStream[] requestShortCircuitFdsForRead(final ExtendedBlock blk, FileInputStream fis[] = new FileInputStream[2]; try { + Preconditions.checkNotNull(data, "Storage not yet initialized"); fis[0] = (FileInputStream)data.getBlockInputStream(blk, 0); fis[1] = DatanodeUtil.getMetaDataInputStream(blk, data); } catch (ClassCastException e) { @@ -3069,6 +3072,7 @@ public static void main(String args[]) { @Override // InterDatanodeProtocol public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock) throws IOException { + Preconditions.checkNotNull(data, "Storage not yet initialized"); return data.initReplicaRecovery(rBlock); } @@ -3079,6 +3083,7 @@ public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock) public String updateReplicaUnderRecovery(final ExtendedBlock oldBlock, final long recoveryId, final long newBlockId, final long newLength) throws IOException { + Preconditions.checkNotNull(data, "Storage not yet initialized"); final Replica r = data.updateReplicaUnderRecovery(oldBlock, recoveryId, newBlockId, newLength); // Notify the namenode of the updated block info. This is important @@ -3360,7 +3365,7 @@ public void deleteBlockPool(String blockPoolId, boolean force) "The block pool is still running. First do a refreshNamenodes to " + "shutdown the block pool service"); } - + Preconditions.checkNotNull(data, "Storage not yet initialized"); data.deleteBlockPool(blockPoolId, force); } @@ -3804,6 +3809,7 @@ public String getSlowDisks() { @Override public List getVolumeReport() throws IOException { checkSuperuserPrivilege(); + Preconditions.checkNotNull(data, "Storage not yet initialized"); Map volumeInfoMap = data.getVolumeInfoMap(); if (volumeInfoMap == null) { LOG.warn("DataNode volume info not available."); From 4ecd77aef07e53693bef67867a21bdea35c441a6 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Thu, 16 Dec 2021 21:27:08 +0800 Subject: [PATCH 22/33] Revert "HDFS-16384. Upgrade Netty to 4.1.72.Final (#3798)" This reverts commit a4557f9ed9af7e0c36a1c668ccccc31e45ad6866. --- LICENSE-binary | 2 +- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 8 -------- hadoop-project/pom.xml | 2 +- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 40207ce0e8a634..6dabe3d71de040 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -261,7 +261,7 @@ io.grpc:grpc-protobuf:1.26.0 io.grpc:grpc-protobuf-lite:1.26.0 io.grpc:grpc-stub:1.26.0 io.netty:netty:3.10.6.Final -io.netty:netty-all:4.1.72.Final +io.netty:netty-all:4.1.42.Final io.netty:netty-buffer:4.1.27.Final io.netty:netty-codec:4.1.27.Final io.netty:netty-codec-http:4.1.27.Final diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index bb4bda5ecc70ff..d65e6030369b3e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -85,10 +85,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> io.netty netty-codec-http - - io.netty - netty-codec-socks - io.netty netty-common @@ -97,10 +93,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> io.netty netty-handler - - io.netty - netty-handler-proxy - io.netty netty-transport diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index e708402617975d..cc45975a3f0771 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -141,7 +141,7 @@ 2.2.4 3.2.4 3.10.6.Final - 4.1.72.Final + 4.1.68.Final 1.1.8.2 1.7.1 From ec06898faae98232451d95c17179d8441cbc3eaf Mon Sep 17 00:00:00 2001 From: Szilard Nemeth <954799+szilard-nemeth@users.noreply.github.com> Date: Thu, 16 Dec 2021 15:53:08 +0100 Subject: [PATCH 23/33] YARN-11048. Add tests that shows how to delete config values with Mutation API (#3799). Contributed by Szilard Nemeth --- .../scheduler/capacity/ParentQueue.java | 6 +- .../scheduler/capacity/QueuePath.java | 11 +- .../resourcemanager/webapp/RMWebServices.java | 78 ++--- .../webapp/TestRMWebServices.java | 2 +- ...estRMWebServicesConfigurationMutation.java | 271 ++++++++++++++++-- 5 files changed, 301 insertions(+), 67 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 283d5678b8c91f..b77a90a3d332a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -344,7 +344,7 @@ void setChildQueues(Collection childQueues) throws IOException { if (Math.abs(childrenPctSum) > PRECISION) { // It is wrong when percent sum != {0, 1} throw new IOException( - "Illegal" + " capacity sum of " + childrenPctSum + "Illegal capacity sum of " + childrenPctSum + " for children of queue " + getQueueName() + " for label=" + nodeLabel + ". It should be either 0 or 1.0"); } else{ @@ -357,7 +357,7 @@ void setChildQueues(Collection childQueues) throws IOException { if ((Math.abs(queueCapacities.getCapacity(nodeLabel)) > PRECISION) && (!allowZeroCapacitySum)) { throw new IOException( - "Illegal" + " capacity sum of " + childrenPctSum + "Illegal capacity sum of " + childrenPctSum + " for children of queue " + getQueueName() + " for label=" + nodeLabel + ". It is set to 0, but parent percent != 0, and " @@ -372,7 +372,7 @@ void setChildQueues(Collection childQueues) throws IOException { queueCapacities.getCapacity(nodeLabel)) <= 0f && !allowZeroCapacitySum) { throw new IOException( - "Illegal" + " capacity sum of " + childrenPctSum + "Illegal capacity sum of " + childrenPctSum + " for children of queue " + getQueueName() + " for label=" + nodeLabel + ". queue=" + getQueueName() + " has zero capacity, but child" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java index 37cfa2ef73366d..440742b908927d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java @@ -61,12 +61,13 @@ public QueuePath(String fullPath) { } /** - * Concatenate queue path parts into one queue path string. - * @param parts Parts of the full queue pathAutoCreatedQueueTemplate - * @return full path of the given queue parts + * Constructor to create Queue path from queue names. + * The provided queue names will be concatenated by dots, giving a full queue path. + * @param parts Parts of queue path + * @return QueuePath object */ - public static String concatenatePath(String... parts) { - return String.join(DOT, parts); + public static QueuePath createFromQueues(String... parts) { + return new QueuePath(String.join(DOT, parts)); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 314b0312081271..041b37c616f041 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -2656,8 +2656,7 @@ public Response formatSchedulerConfiguration(@Context HttpServletRequest hsr) initForWritableEndpoints(callerUGI, true); ResourceScheduler scheduler = rm.getResourceScheduler(); - if (scheduler instanceof MutableConfScheduler - && ((MutableConfScheduler) scheduler).isConfigurationMutable()) { + if (isConfigurationMutable(scheduler)) { try { MutableConfigurationProvider mutableConfigurationProvider = ((MutableConfScheduler) scheduler).getMutableConfProvider(); @@ -2696,8 +2695,7 @@ public synchronized Response validateAndGetSchedulerConfiguration( UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); initForWritableEndpoints(callerUGI, true); ResourceScheduler scheduler = rm.getResourceScheduler(); - if (scheduler instanceof MutableConfScheduler && ((MutableConfScheduler) - scheduler).isConfigurationMutable()) { + if (isConfigurationMutable(scheduler)) { try { MutableConfigurationProvider mutableConfigurationProvider = ((MutableConfScheduler) scheduler).getMutableConfProvider(); @@ -2746,51 +2744,61 @@ public synchronized Response validateAndGetSchedulerConfiguration( public synchronized Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, @Context HttpServletRequest hsr) throws AuthorizationException, InterruptedException { - UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); initForWritableEndpoints(callerUGI, true); ResourceScheduler scheduler = rm.getResourceScheduler(); - if (scheduler instanceof MutableConfScheduler && ((MutableConfScheduler) - scheduler).isConfigurationMutable()) { + if (isConfigurationMutable(scheduler)) { try { - callerUGI.doAs(new PrivilegedExceptionAction() { - @Override - public Void run() throws Exception { - MutableConfigurationProvider provider = ((MutableConfScheduler) - scheduler).getMutableConfProvider(); - if (!provider.getAclMutationPolicy().isMutationAllowed(callerUGI, - mutationInfo)) { - throw new org.apache.hadoop.security.AccessControlException("User" - + " is not admin of all modified queues."); - } - LogMutation logMutation = provider.logAndApplyMutation(callerUGI, - mutationInfo); - try { - rm.getRMContext().getRMAdminService().refreshQueues(); - } catch (IOException | YarnException e) { - provider.confirmPendingMutation(logMutation, false); - throw e; - } - provider.confirmPendingMutation(logMutation, true); - return null; - } + callerUGI.doAs((PrivilegedExceptionAction) () -> { + MutableConfigurationProvider provider = ((MutableConfScheduler) + scheduler).getMutableConfProvider(); + LogMutation logMutation = applyMutation(provider, callerUGI, mutationInfo); + return refreshQueues(provider, logMutation); }); } catch (IOException e) { LOG.error("Exception thrown when modifying configuration.", e); return Response.status(Status.BAD_REQUEST).entity(e.getMessage()) .build(); } - return Response.status(Status.OK).entity("Configuration change " + - "successfully applied.").build(); + return Response.status(Status.OK).entity("Configuration change successfully applied.") + .build(); } else { return Response.status(Status.BAD_REQUEST) - .entity("Configuration change only supported by " + - "MutableConfScheduler.") + .entity(String.format("Configuration change only supported by " + + "%s.", MutableConfScheduler.class.getSimpleName())) .build(); } } + private Void refreshQueues(MutableConfigurationProvider provider, LogMutation logMutation) + throws Exception { + try { + rm.getRMContext().getRMAdminService().refreshQueues(); + } catch (IOException | YarnException e) { + provider.confirmPendingMutation(logMutation, false); + throw e; + } + provider.confirmPendingMutation(logMutation, true); + return null; + } + + private LogMutation applyMutation(MutableConfigurationProvider provider, + UserGroupInformation callerUGI, SchedConfUpdateInfo mutationInfo) throws Exception { + if (!provider.getAclMutationPolicy().isMutationAllowed(callerUGI, + mutationInfo)) { + throw new org.apache.hadoop.security.AccessControlException("User" + + " is not admin of all modified queues."); + } + return provider.logAndApplyMutation(callerUGI, + mutationInfo); + } + + private boolean isConfigurationMutable(ResourceScheduler scheduler) { + return scheduler instanceof MutableConfScheduler && ((MutableConfScheduler) + scheduler).isConfigurationMutable(); + } + @GET @Path(RMWSConsts.SCHEDULER_CONF) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, @@ -2803,8 +2811,7 @@ public Response getSchedulerConfiguration(@Context HttpServletRequest hsr) initForWritableEndpoints(callerUGI, true); ResourceScheduler scheduler = rm.getResourceScheduler(); - if (scheduler instanceof MutableConfScheduler - && ((MutableConfScheduler) scheduler).isConfigurationMutable()) { + if (isConfigurationMutable(scheduler)) { MutableConfigurationProvider mutableConfigurationProvider = ((MutableConfScheduler) scheduler).getMutableConfProvider(); // We load the cached configuration from configuration store, @@ -2835,8 +2842,7 @@ public Response getSchedulerConfigurationVersion(@Context initForWritableEndpoints(callerUGI, true); ResourceScheduler scheduler = rm.getResourceScheduler(); - if (scheduler instanceof MutableConfScheduler - && ((MutableConfScheduler) scheduler).isConfigurationMutable()) { + if (isConfigurationMutable(scheduler)) { MutableConfigurationProvider mutableConfigurationProvider = ((MutableConfScheduler) scheduler).getMutableConfProvider(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 673fbbe2ec079b..d5d534395b54b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -1098,4 +1098,4 @@ private RMWebServices prepareWebServiceForValidation( return webService; } -} +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 15599863d064a1..675e79243f4227 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -22,10 +22,13 @@ import com.google.inject.servlet.ServletModule; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.core.util.MultivaluedMapImpl; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.test.framework.WebAppDescriptor; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Sets; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; @@ -34,12 +37,13 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; -import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -58,10 +62,15 @@ import java.util.HashMap; import java.util.Map; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_CAPACITY; +import static org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils.toJson; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertNotNull; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY; +import static org.junit.Assert.assertTrue; /** * Test scheduler configuration mutation via REST API. @@ -74,7 +83,11 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE); private static final File OLD_CONF_FILE = new File(new File("target", "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE + ".tmp"); - + private static final String LABEL_1 = "label1"; + public static final QueuePath ROOT = new QueuePath("root"); + public static final QueuePath ROOT_A = new QueuePath("root", "a"); + public static final QueuePath ROOT_A_A1 = QueuePath.createFromQueues("root", "a", "a1"); + public static final QueuePath ROOT_A_A2 = QueuePath.createFromQueues("root", "a", "a2"); private static MockRM rm; private static String userName; private static CapacitySchedulerConfiguration csConf; @@ -216,7 +229,7 @@ public void testFormatSchedulerConf() throws Exception { ClientResponse response = r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); newConf = getSchedulerConf(); @@ -284,7 +297,7 @@ public void testAddNestedQueue() throws Exception { r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -323,7 +336,7 @@ public void testAddWithUpdate() throws Exception { r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -350,7 +363,7 @@ public void testUnsetParentQueueOrderingPolicy() throws Exception { response = r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo1, + .entity(toJson(updateInfo1, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -371,7 +384,7 @@ public void testUnsetParentQueueOrderingPolicy() throws Exception { response = r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo2, + .entity(toJson(updateInfo2, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -401,7 +414,7 @@ public void testUnsetLeafQueueOrderingPolicy() throws Exception { response = r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo1, + .entity(toJson(updateInfo1, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -419,7 +432,7 @@ public void testUnsetLeafQueueOrderingPolicy() throws Exception { response = r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo2, + .entity(toJson(updateInfo2, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -448,7 +461,7 @@ public void testRemoveQueue() throws Exception { r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -480,7 +493,7 @@ public void testStopWithRemoveQueue() throws Exception { response = r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -513,7 +526,7 @@ public void testStopWithConvertLeafToParentQueue() throws Exception { response = r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -538,7 +551,7 @@ public void testRemoveParentQueue() throws Exception { r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -569,7 +582,7 @@ public void testRemoveParentQueueWithCapacity() throws Exception { r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -601,7 +614,7 @@ public void testRemoveMultipleQueues() throws Exception { r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -629,7 +642,7 @@ private void stopQueue(String... queuePaths) throws Exception { r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -664,7 +677,7 @@ public void testUpdateQueue() throws Exception { r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); LOG.debug("Response headers: " + response.getHeaders()); @@ -683,7 +696,7 @@ public void testUpdateQueue() throws Exception { r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -713,7 +726,7 @@ public void testUpdateQueueCapacity() throws Exception { r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -738,7 +751,7 @@ public void testGlobalConfChange() throws Exception { r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -753,7 +766,7 @@ public void testGlobalConfChange() throws Exception { r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -764,6 +777,220 @@ public void testGlobalConfChange() throws Exception { newCSConf.getMaximumSystemApplications()); } + @Test + public void testNodeLabelRemovalResidualConfigsAreCleared() throws Exception { + WebResource r = resource(); + ClientResponse response; + + // 1. Create Node Label: label1 + NodeLabelsInfo nodeLabelsInfo = new NodeLabelsInfo(); + nodeLabelsInfo.getNodeLabelsInfo().add(new NodeLabelInfo(LABEL_1)); + WebResource addNodeLabelsResource = r.path("ws").path("v1").path("cluster") + .path("add-node-labels"); + WebResource getNodeLabelsResource = r.path("ws").path("v1").path("cluster") + .path("get-node-labels"); + WebResource removeNodeLabelsResource = r.path("ws").path("v1").path("cluster") + .path("remove-node-labels"); + WebResource schedulerConfResource = r.path("ws").path("v1").path("cluster") + .path(RMWSConsts.SCHEDULER_CONF); + response = + addNodeLabelsResource.queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(logAndReturnJson(addNodeLabelsResource, + toJson(nodeLabelsInfo, NodeLabelsInfo.class)), + MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + + // 2. Verify new Node Label + response = + getNodeLabelsResource.queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + nodeLabelsInfo = response.getEntity(NodeLabelsInfo.class); + assertEquals(1, nodeLabelsInfo.getNodeLabels().size()); + for (NodeLabelInfo nl : nodeLabelsInfo.getNodeLabelsInfo()) { + assertEquals(LABEL_1, nl.getName()); + assertTrue(nl.getExclusivity()); + } + + // 3. Assign 'label1' to root.a + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); + Map updateForRoot = new HashMap<>(); + updateForRoot.put(CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS, "*"); + QueueConfigInfo rootUpdateInfo = new QueueConfigInfo(ROOT.getFullPath(), updateForRoot); + + Map updateForRootA = new HashMap<>(); + updateForRootA.put(CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS, LABEL_1); + QueueConfigInfo rootAUpdateInfo = new QueueConfigInfo(ROOT_A.getFullPath(), updateForRootA); + + updateInfo.getUpdateQueueInfo().add(rootUpdateInfo); + updateInfo.getUpdateQueueInfo().add(rootAUpdateInfo); + + response = + schedulerConfResource + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(logAndReturnJson(schedulerConfResource, toJson(updateInfo, + SchedConfUpdateInfo.class)), MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + assertEquals(Sets.newHashSet("*"), + cs.getConfiguration().getAccessibleNodeLabels(ROOT.getFullPath())); + assertEquals(Sets.newHashSet(LABEL_1), + cs.getConfiguration().getAccessibleNodeLabels(ROOT_A.getFullPath())); + + // 4. Set partition capacities to queues as below + updateInfo = new SchedConfUpdateInfo(); + updateForRoot = new HashMap<>(); + updateForRoot.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), "100"); + updateForRoot.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), "100"); + rootUpdateInfo = new QueueConfigInfo(ROOT.getFullPath(), updateForRoot); + + updateForRootA = new HashMap<>(); + updateForRootA.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), "100"); + updateForRootA.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), "100"); + rootAUpdateInfo = new QueueConfigInfo(ROOT_A.getFullPath(), updateForRootA); + + // Avoid the following exception by adding some capacities to root.a.a1 and root.a.a2 to label1 + // Illegal capacity sum of 0.0 for children of queue a for label=label1. + // It is set to 0, but parent percent != 0, and doesn't allow children capacity to set to 0 + Map updateForRootA_A1 = new HashMap<>(); + updateForRootA_A1.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), "20"); + updateForRootA_A1.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), "20"); + QueueConfigInfo rootA_A1UpdateInfo = new QueueConfigInfo(ROOT_A_A1.getFullPath(), + updateForRootA_A1); + + Map updateForRootA_A2 = new HashMap<>(); + updateForRootA_A2.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), "80"); + updateForRootA_A2.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), "80"); + QueueConfigInfo rootA_A2UpdateInfo = new QueueConfigInfo(ROOT_A_A2.getFullPath(), + updateForRootA_A2); + + + updateInfo.getUpdateQueueInfo().add(rootUpdateInfo); + updateInfo.getUpdateQueueInfo().add(rootAUpdateInfo); + updateInfo.getUpdateQueueInfo().add(rootA_A1UpdateInfo); + updateInfo.getUpdateQueueInfo().add(rootA_A2UpdateInfo); + + response = + schedulerConfResource + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(logAndReturnJson(schedulerConfResource, toJson(updateInfo, + SchedConfUpdateInfo.class)), MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + + assertEquals(100.0, cs.getConfiguration().getLabeledQueueCapacity(ROOT, LABEL_1), 0.001f); + assertEquals(100.0, cs.getConfiguration().getLabeledQueueMaximumCapacity(ROOT, LABEL_1), + 0.001f); + assertEquals(100.0, cs.getConfiguration().getLabeledQueueCapacity(ROOT_A, LABEL_1), 0.001f); + assertEquals(100.0, cs.getConfiguration().getLabeledQueueMaximumCapacity(ROOT_A, LABEL_1), + 0.001f); + assertEquals(20.0, cs.getConfiguration().getLabeledQueueCapacity(ROOT_A_A1, LABEL_1), 0.001f); + assertEquals(20.0, cs.getConfiguration().getLabeledQueueMaximumCapacity(ROOT_A_A1, LABEL_1), + 0.001f); + assertEquals(80.0, cs.getConfiguration().getLabeledQueueCapacity(ROOT_A_A2, LABEL_1), 0.001f); + assertEquals(80.0, cs.getConfiguration().getLabeledQueueMaximumCapacity(ROOT_A_A2, LABEL_1), + 0.001f); + + //5. De-assign node label: "label1" + Remove residual properties + updateInfo = new SchedConfUpdateInfo(); + updateForRoot = new HashMap<>(); + updateForRoot.put(CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS, "*"); + updateForRoot.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), ""); + updateForRoot.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), ""); + rootUpdateInfo = new QueueConfigInfo(ROOT.getFullPath(), updateForRoot); + + updateForRootA = new HashMap<>(); + updateForRootA.put(CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS, ""); + updateForRootA.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), ""); + updateForRootA.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), ""); + rootAUpdateInfo = new QueueConfigInfo(ROOT_A.getFullPath(), updateForRootA); + + updateForRootA_A1 = new HashMap<>(); + updateForRootA_A1.put(CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS, ""); + updateForRootA_A1.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), ""); + updateForRootA_A1.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), ""); + rootA_A1UpdateInfo = new QueueConfigInfo(ROOT_A_A1.getFullPath(), updateForRootA_A1); + + updateForRootA_A2 = new HashMap<>(); + updateForRootA_A2.put(CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS, ""); + updateForRootA_A2.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), ""); + updateForRootA_A2.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), ""); + rootA_A2UpdateInfo = new QueueConfigInfo(ROOT_A_A2.getFullPath(), updateForRootA_A2); + + updateInfo.getUpdateQueueInfo().add(rootUpdateInfo); + updateInfo.getUpdateQueueInfo().add(rootAUpdateInfo); + updateInfo.getUpdateQueueInfo().add(rootA_A1UpdateInfo); + updateInfo.getUpdateQueueInfo().add(rootA_A2UpdateInfo); + + response = + schedulerConfResource + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(logAndReturnJson(schedulerConfResource, toJson(updateInfo, + SchedConfUpdateInfo.class)), MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + assertEquals(Sets.newHashSet("*"), + cs.getConfiguration().getAccessibleNodeLabels(ROOT.getFullPath())); + assertNull(cs.getConfiguration().getAccessibleNodeLabels(ROOT_A.getFullPath())); + + //6. Remove node label 'label1' + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("labels", LABEL_1); + response = + removeNodeLabelsResource + .queryParam("user.name", userName) + .queryParams(params) + .accept(MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + + // Verify + response = + getNodeLabelsResource.queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + nodeLabelsInfo = response.getEntity(NodeLabelsInfo.class); + assertEquals(0, nodeLabelsInfo.getNodeLabels().size()); + + //6. Check residual configs + assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT, LABEL_1, CAPACITY)); + assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT, LABEL_1, MAXIMUM_CAPACITY)); + assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT_A, LABEL_1, CAPACITY)); + assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT_A, LABEL_1, MAXIMUM_CAPACITY)); + assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT_A_A1, LABEL_1, CAPACITY)); + assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT_A_A1, LABEL_1, MAXIMUM_CAPACITY)); + assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT_A_A2, LABEL_1, CAPACITY)); + assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT_A_A2, LABEL_1, MAXIMUM_CAPACITY)); + } + + private String getConfValueForQueueAndLabelAndType(CapacityScheduler cs, + QueuePath queuePath, String label, String type) { + return cs.getConfiguration().get( + CapacitySchedulerConfiguration.getNodeLabelPrefix( + queuePath.getFullPath(), label) + type); + } + + private Object logAndReturnJson(WebResource ws, String json) { + LOG.info("Sending to web resource: {}, json: {}", ws, json); + return json; + } + + private String getAccessibleNodeLabelsCapacityPropertyName(String label) { + return String.format("%s.%s.%s", ACCESSIBLE_NODE_LABELS, label, CAPACITY); + } + + private String getAccessibleNodeLabelsMaxCapacityPropertyName(String label) { + return String.format("%s.%s.%s", ACCESSIBLE_NODE_LABELS, label, MAXIMUM_CAPACITY); + } + @Test public void testValidateWithClusterMaxAllocation() throws Exception { WebResource r = resource(); @@ -784,7 +1011,7 @@ public void testValidateWithClusterMaxAllocation() throws Exception { .path(RMWSConsts.SCHEDULER_CONF_VALIDATE) .queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(YarnWebServiceUtils.toJson(updateInfo, + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .post(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); From 9d7c3c4442ed310a6d85d8c52fb2ba3511e7b690 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Thu, 16 Dec 2021 23:39:18 +0100 Subject: [PATCH 24/33] YARN-10963. Split TestCapacityScheduler by test categories. Contributed by Tamas Domok --- ...pacitySchedulerConfigGeneratorForTest.java | 52 + .../CapacitySchedulerTestUtilities.java | 149 + .../capacity/TestCapacityScheduler.java | 2889 +---------------- .../capacity/TestCapacitySchedulerApps.java | 1499 +++++++++ .../capacity/TestCapacitySchedulerNodes.java | 387 +++ .../capacity/TestCapacitySchedulerQueues.java | 888 +++++ 6 files changed, 3046 insertions(+), 2818 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java index 873e3b95a4ca28..087b797f2d84cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java @@ -18,7 +18,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import java.util.HashMap; import java.util.Map; @@ -50,4 +54,52 @@ public static Configuration createBasicCSConfiguration() { return createConfiguration(conf); } + public static void setMinAllocMb(Configuration conf, int minAllocMb) { + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + minAllocMb); + } + + public static void setMaxAllocMb(Configuration conf, int maxAllocMb) { + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + maxAllocMb); + } + + public static void setMaxAllocMb(CapacitySchedulerConfiguration conf, + String queueName, int maxAllocMb) { + String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) + + MAXIMUM_ALLOCATION_MB; + conf.setInt(propName, maxAllocMb); + } + + public static void setMinAllocVcores(Configuration conf, int minAllocVcores) { + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + minAllocVcores); + } + + public static void setMaxAllocVcores(Configuration conf, int maxAllocVcores) { + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + maxAllocVcores); + } + + public static void setMaxAllocVcores(CapacitySchedulerConfiguration conf, + String queueName, int maxAllocVcores) { + String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) + + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES; + conf.setInt(propName, maxAllocVcores); + } + + public static void setMaxAllocation(CapacitySchedulerConfiguration conf, + String queueName, String maxAllocation) { + String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) + + MAXIMUM_ALLOCATION; + conf.set(propName, maxAllocation); + } + + public static void unsetMaxAllocation(CapacitySchedulerConfiguration conf, + String queueName) { + String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) + + MAXIMUM_ALLOCATION; + conf.unset(propName); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerTestUtilities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerTestUtilities.java index 3d098f837a8fdf..b2c654891ebfcf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerTestUtilities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerTestUtilities.java @@ -18,14 +18,48 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.util.Sets; +import org.apache.hadoop.yarn.LocalConfigurationProvider; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.api.records.NodeStatus; +import org.apache.hadoop.yarn.server.resourcemanager.Application; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.NodeManager; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.junit.Assert; +import java.io.IOException; import java.util.Set; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public final class CapacitySchedulerTestUtilities { public static final int GB = 1024; @@ -69,4 +103,119 @@ public static void waitforNMRegistered(ResourceScheduler scheduler, int nodecoun } } } + + public static ResourceManager createResourceManager() throws Exception { + ResourceUtils.resetResourceTypes(new Configuration()); + DefaultMetricsSystem.setMiniClusterMode(true); + ResourceManager resourceManager = new ResourceManager() { + @Override + protected RMNodeLabelsManager createNodeLabelManager() { + RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); + mgr.init(getConfig()); + return mgr; + } + }; + CapacitySchedulerConfiguration csConf + = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(csConf); + YarnConfiguration conf = new YarnConfiguration(csConf); + conf.setClass(YarnConfiguration.RM_SCHEDULER, + CapacityScheduler.class, ResourceScheduler.class); + resourceManager.init(conf); + resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); + resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); + ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start(); + return resourceManager; + } + + public static RMContext createMockRMContext() { + RMContext mockContext = mock(RMContext.class); + when(mockContext.getConfigurationProvider()).thenReturn( + new LocalConfigurationProvider()); + return mockContext; + } + + public static void stopResourceManager(ResourceManager resourceManager) throws Exception { + if (resourceManager != null) { + QueueMetrics.clearQueueMetrics(); + DefaultMetricsSystem.shutdown(); + resourceManager.stop(); + } + } + + public static ApplicationAttemptId appHelper(MockRM rm, CapacityScheduler cs, + int clusterTs, int appId, String queue, + String user) { + ApplicationId appId1 = BuilderUtils.newApplicationId(clusterTs, appId); + ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId( + appId1, appId); + + RMAppAttemptMetrics attemptMetric1 = + new RMAppAttemptMetrics(appAttemptId1, rm.getRMContext()); + RMAppImpl app1 = mock(RMAppImpl.class); + when(app1.getApplicationId()).thenReturn(appId1); + RMAppAttemptImpl attempt1 = mock(RMAppAttemptImpl.class); + Container container = mock(Container.class); + when(attempt1.getMasterContainer()).thenReturn(container); + ApplicationSubmissionContext submissionContext = mock( + ApplicationSubmissionContext.class); + when(attempt1.getSubmissionContext()).thenReturn(submissionContext); + when(attempt1.getAppAttemptId()).thenReturn(appAttemptId1); + when(attempt1.getRMAppAttemptMetrics()).thenReturn(attemptMetric1); + when(app1.getCurrentAppAttempt()).thenReturn(attempt1); + rm.getRMContext().getRMApps().put(appId1, app1); + + SchedulerEvent addAppEvent1 = + new AppAddedSchedulerEvent(appId1, queue, user); + cs.handle(addAppEvent1); + SchedulerEvent addAttemptEvent1 = + new AppAttemptAddedSchedulerEvent(appAttemptId1, false); + cs.handle(addAttemptEvent1); + return appAttemptId1; + } + + public static MockRM setUpMove() { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + return setUpMove(conf); + } + + public static MockRM setUpMove(Configuration config) { + CapacitySchedulerConfiguration conf = + new CapacitySchedulerConfiguration(config); + setupQueueConfiguration(conf); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + MockRM rm = new MockRM(conf); + rm.start(); + return rm; + } + + public static void nodeUpdate(ResourceManager rm, NodeManager nm) { + RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId()); + // Send a heartbeat to kick the tires on the Scheduler + NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); + rm.getResourceScheduler().handle(nodeUpdate); + } + + public static NodeManager registerNode(ResourceManager rm, String hostName, + int containerManagerPort, int httpPort, String rackName, + Resource capability, NodeStatus nodeStatus) + throws IOException, YarnException { + NodeManager nm = new NodeManager(hostName, + containerManagerPort, httpPort, rackName, capability, rm, nodeStatus); + NodeAddedSchedulerEvent nodeAddEvent1 = + new NodeAddedSchedulerEvent(rm.getRMContext().getRMNodes() + .get(nm.getNodeId())); + rm.getResourceScheduler().handle(nodeAddEvent1); + return nm; + } + + public static void checkApplicationResourceUsage(int expected, Application application) { + Assert.assertEquals(expected, application.getUsedResources().getMemorySize()); + } + + public static void checkNodeResourceUsage(int expected, NodeManager node) { + Assert.assertEquals(expected, node.getUsed().getMemorySize()); + node.checkResourceUsage(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index c3548cc6f7ecbf..4a9e45e756f28e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -19,13 +19,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import static org.apache.hadoop.yarn.server.resourcemanager.MockNM.createMockNodeStatus; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.checkQueueStructureCapacities; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocMb; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocVcores; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMinAllocMb; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMinAllocVcores; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.findQueue; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.getDefaultCapacities; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ExpectedCapacities; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupBlockedQueueConfiguration; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupOtherBlockedQueueConfiguration; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfWithoutChildrenOfB; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1; @@ -35,22 +35,22 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_CAPACITY; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3_CAPACITY; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithB1AsParentQueue; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithoutB; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithoutB1; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.appHelper; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkApplicationResourceUsage; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkNodeResourceUsage; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkPendingResource; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkPendingResourceGreaterThanZero; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createMockRMContext; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createResourceManager; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.nodeUpdate; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.registerNode; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.setUpMove; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.stopResourceManager; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.toSet; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.waitforNMRegistered; import static org.assertj.core.api.Assertions.assertThat; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerOvercommit.assertContainerKilled; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerOvercommit.assertMemory; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerOvercommit.assertNoPreemption; @@ -61,16 +61,11 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.io.IOException; import java.net.InetSocketAddress; import java.security.PrivilegedAction; import java.util.ArrayList; @@ -90,7 +85,6 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.Groups; @@ -109,7 +103,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -119,11 +112,9 @@ import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.PreemptionMessage; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; -import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; @@ -131,10 +122,6 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.Event; -import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -159,7 +146,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics; @@ -167,24 +153,15 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.CSQueueMetricsForCustomResources; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestQueueMetricsForCustomResources; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.AllocationState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.ContainerAllocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest; @@ -199,16 +176,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimpleCandidateNodeSet; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.IteratorSelector; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; @@ -239,51 +211,13 @@ public class TestCapacityScheduler { @Before public void setUp() throws Exception { - ResourceUtils.resetResourceTypes(new Configuration()); - DefaultMetricsSystem.setMiniClusterMode(true); - resourceManager = new ResourceManager() { - @Override - protected RMNodeLabelsManager createNodeLabelManager() { - RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); - mgr.init(getConfig()); - return mgr; - } - }; - CapacitySchedulerConfiguration csConf - = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(csConf); - YarnConfiguration conf = new YarnConfiguration(csConf); - conf.setClass(YarnConfiguration.RM_SCHEDULER, - CapacityScheduler.class, ResourceScheduler.class); - resourceManager.init(conf); - resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); - resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); - ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); - mockContext = mock(RMContext.class); - when(mockContext.getConfigurationProvider()).thenReturn( - new LocalConfigurationProvider()); + resourceManager = createResourceManager(); + mockContext = createMockRMContext(); } @After public void tearDown() throws Exception { - if (resourceManager != null) { - QueueMetrics.clearQueueMetrics(); - DefaultMetricsSystem.shutdown(); - resourceManager.stop(); - } - } - - private NodeManager registerNode(ResourceManager rm, String hostName, - int containerManagerPort, int httpPort, String rackName, - Resource capability, NodeStatus nodeStatus) - throws IOException, YarnException { - NodeManager nm = new NodeManager(hostName, - containerManagerPort, httpPort, rackName, capability, rm, nodeStatus); - NodeAddedSchedulerEvent nodeAddEvent1 = - new NodeAddedSchedulerEvent(rm.getRMContext().getRMNodes() - .get(nm.getNodeId())); - rm.getResourceScheduler().handle(nodeAddEvent1); - return nm; + stopResourceManager(resourceManager); } @Test (timeout = 30000) @@ -291,8 +225,9 @@ public void testConfValidation() throws Exception { CapacityScheduler scheduler = new CapacityScheduler(); scheduler.setRMContext(resourceManager.getRMContext()); Configuration conf = new YarnConfiguration(); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); + + setMinAllocMb(conf, 2048); + setMaxAllocMb(conf, 1024); try { scheduler.init(conf); fail("Exception is expected because the min memory allocation is" + @@ -305,8 +240,8 @@ public void testConfValidation() throws Exception { } conf = new YarnConfiguration(); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 1); + setMinAllocVcores(conf, 2); + setMaxAllocVcores(conf, 1); try { scheduler.reinitialize(conf, mockContext); fail("Exception is expected because the min vcores allocation is" + @@ -319,19 +254,6 @@ public void testConfValidation() throws Exception { } } - private NodeManager registerNode(String hostName, int containerManagerPort, - int httpPort, String rackName, - Resource capability, NodeStatus nodeStatus) - throws IOException, YarnException { - NodeManager nm = new NodeManager(hostName, containerManagerPort, httpPort, - rackName, capability, resourceManager, nodeStatus); - NodeAddedSchedulerEvent nodeAddEvent1 = - new NodeAddedSchedulerEvent(resourceManager.getRMContext() - .getRMNodes().get(nm.getNodeId())); - resourceManager.getResourceScheduler().handle(nodeAddEvent1); - return nm; - } - @Test public void testCapacityScheduler() throws Exception { @@ -342,13 +264,13 @@ public void testCapacityScheduler() throws Exception { // Register node1 String host_0 = "host_0"; NodeManager nm_0 = - registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, + registerNode(resourceManager, host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(4 * GB, 1), mockNodeStatus); // Register node2 String host_1 = "host_1"; NodeManager nm_1 = - registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, + registerNode(resourceManager, host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(2 * GB, 1), mockNodeStatus); // ResourceRequest priorities @@ -397,10 +319,10 @@ public void testCapacityScheduler() throws Exception { LOG.info("Kick!"); // task_0_0 and task_1_0 allocated, used=4G - nodeUpdate(nm_0); + nodeUpdate(resourceManager, nm_0); // nothing allocated - nodeUpdate(nm_1); + nodeUpdate(resourceManager, nm_1); // Get allocations from the scheduler application_0.schedule(); // task_0_0 @@ -429,11 +351,11 @@ public void testCapacityScheduler() throws Exception { // Send a heartbeat to kick the tires on the Scheduler LOG.info("Sending hb from " + nm_0.getHostName()); // nothing new, used=4G - nodeUpdate(nm_0); + nodeUpdate(resourceManager, nm_0); LOG.info("Sending hb from " + nm_1.getHostName()); // task_0_1 is prefer as locality, used=2G - nodeUpdate(nm_1); + nodeUpdate(resourceManager, nm_1); // Get allocations from the scheduler LOG.info("Trying to allocate..."); @@ -443,8 +365,8 @@ public void testCapacityScheduler() throws Exception { application_1.schedule(); checkApplicationResourceUsage(5 * GB, application_1); - nodeUpdate(nm_0); - nodeUpdate(nm_1); + nodeUpdate(resourceManager, nm_0); + nodeUpdate(resourceManager, nm_1); checkNodeResourceUsage(4*GB, nm_0); checkNodeResourceUsage(2*GB, nm_1); @@ -658,21 +580,6 @@ protected RMNodeLabelsManager createNodeLabelManager() { LOG.info("--- END: testAssignMultiple ---"); } - private void nodeUpdate(ResourceManager rm, NodeManager nm) { - RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId()); - // Send a heartbeat to kick the tires on the Scheduler - NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); - rm.getResourceScheduler().handle(nodeUpdate); - } - - private void nodeUpdate(NodeManager nm) { - RMNode node = resourceManager.getRMContext().getRMNodes().get(nm.getNodeId()); - // Send a heartbeat to kick the tires on the Scheduler - NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); - resourceManager.getResourceScheduler().handle(nodeUpdate); - } - - @Test public void testMaximumCapacitySetup() { float delta = 0.0000001f; @@ -695,10 +602,8 @@ public void testQueueMaximumAllocations() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A1) - + MAXIMUM_ALLOCATION_MB, "1024"); - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A1) - + MAXIMUM_ALLOCATION_VCORES, "1"); + setMaxAllocMb(conf, A1, 1024); + setMaxAllocVcores(conf, A1, 1); scheduler.init(conf); scheduler.start(); @@ -722,64 +627,6 @@ public void testQueueMaximumAllocations() { Assert.assertEquals(1, maxAllocationForQueue.getVirtualCores()); } - - @Test - public void testRefreshQueues() throws Exception { - CapacityScheduler cs = new CapacityScheduler(); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null); - setupQueueConfiguration(conf); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - cs.init(conf); - cs.start(); - cs.reinitialize(conf, rmContext); - checkQueueStructureCapacities(cs); - - conf.setCapacity(A, 80f); - conf.setCapacity(B, 20f); - cs.reinitialize(conf, mockContext); - checkQueueStructureCapacities(cs, getDefaultCapacities(80f / 100.0f, 20f / 100.0f)); - cs.stop(); - } - - private void checkApplicationResourceUsage(int expected, - Application application) { - Assert.assertEquals(expected, application.getUsedResources().getMemorySize()); - } - - private void checkNodeResourceUsage(int expected, NodeManager node) { - Assert.assertEquals(expected, node.getUsed().getMemorySize()); - node.checkResourceUsage(); - } - - /** Test that parseQueue throws an exception when two leaf queues have the - * same name - * @throws IOException - */ - @Test(expected=IOException.class) - public void testParseQueue() throws IOException { - CapacityScheduler cs = new CapacityScheduler(); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); - cs.init(conf); - cs.start(); - - conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} ); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); - conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); - - cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null)); - } - @Test public void testParseQueueWithAbsoluteResource() { String childQueue = "testQueue"; @@ -819,85 +666,6 @@ public void testParseQueueWithAbsoluteResource() { assertEquals(10, childQueueLabelCapacity.getVirtualCores()); } - @Test - public void testReconnectedNode() throws Exception { - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); - setupQueueConfiguration(csConf); - CapacityScheduler cs = new CapacityScheduler(); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - cs.init(csConf); - cs.start(); - cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, - null, null, new RMContainerTokenSecretManager(csConf), - new NMTokenSecretManagerInRM(csConf), - new ClientToAMTokenSecretManagerInRM(), null)); - - RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1); - RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2); - - cs.handle(new NodeAddedSchedulerEvent(n1)); - cs.handle(new NodeAddedSchedulerEvent(n2)); - - Assert.assertEquals(6 * GB, cs.getClusterResource().getMemorySize()); - - // reconnect n1 with downgraded memory - n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1); - cs.handle(new NodeRemovedSchedulerEvent(n1)); - cs.handle(new NodeAddedSchedulerEvent(n1)); - - Assert.assertEquals(4 * GB, cs.getClusterResource().getMemorySize()); - cs.stop(); - } - - @Test - public void testRefreshQueuesWithNewQueue() throws Exception { - CapacityScheduler cs = new CapacityScheduler(); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - cs.init(conf); - cs.start(); - cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null)); - checkQueueStructureCapacities(cs); - - // Add a new queue b4 - final String b4 = B + ".b4"; - final float b4Capacity = 10; - final float modifiedB3Capacity = B3_CAPACITY - b4Capacity; - - try { - conf.setCapacity(A, 80f); - conf.setCapacity(B, 20f); - conf.setQueues(B, new String[]{"b1", "b2", "b3", "b4"}); - conf.setCapacity(B1, B1_CAPACITY); - conf.setCapacity(B2, B2_CAPACITY); - conf.setCapacity(B3, modifiedB3Capacity); - conf.setCapacity(b4, b4Capacity); - cs.reinitialize(conf, mockContext); - - final float capA = 80f / 100.0f; - final float capB = 20f / 100.0f; - Map expectedCapacities = getDefaultCapacities(capA, capB); - expectedCapacities.put(B3, new ExpectedCapacities(modifiedB3Capacity / 100.0f, capB)); - expectedCapacities.put(b4, new ExpectedCapacities(b4Capacity / 100.0f, capB)); - checkQueueStructureCapacities(cs, expectedCapacities); - - // Verify parent for B4 - CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueB = findQueue(rootQueue, B); - CSQueue queueB4 = findQueue(queueB, b4); - - assertEquals(queueB, queueB4.getParent()); - } finally { - cs.stop(); - } - } @Test public void testCapacitySchedulerInfo() throws Exception { QueueInfo queueInfo = resourceManager.getResourceScheduler().getQueueInfo("a", true, true); @@ -927,36 +695,6 @@ private int getQueueCount(List queueInformation, String queueN return result; } - @Test - public void testBlackListNodes() throws Exception { - Configuration conf = new Configuration(); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); - MockRM rm = new MockRM(conf); - rm.start(); - CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); - - String host = "127.0.0.1"; - RMNode node = - MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host); - cs.handle(new NodeAddedSchedulerEvent(node)); - - ApplicationAttemptId appAttemptId = appHelper(rm, cs, 100, 1, "default", "user"); - - // Verify the blacklist can be updated independent of requesting containers - cs.allocate(appAttemptId, Collections.emptyList(), null, - Collections.emptyList(), - Collections.singletonList(host), null, NULL_UPDATE_REQUESTS); - Assert.assertTrue(cs.getApplicationAttempt(appAttemptId) - .isPlaceBlacklisted(host)); - cs.allocate(appAttemptId, Collections.emptyList(), null, - Collections.emptyList(), null, - Collections.singletonList(host), NULL_UPDATE_REQUESTS); - Assert.assertFalse(cs.getApplicationAttempt(appAttemptId) - .isPlaceBlacklisted(host)); - rm.stop(); - } - @Test public void testAllocateReorder() throws Exception { @@ -1162,53 +900,6 @@ public void testResourceOverCommit() throws Exception { rm.stop(); } - @Test - public void testGetAppsInQueue() throws Exception { - Application application_0 = new Application("user_0", "a1", resourceManager); - application_0.submit(); - - Application application_1 = new Application("user_0", "a2", resourceManager); - application_1.submit(); - - Application application_2 = new Application("user_0", "b2", resourceManager); - application_2.submit(); - - ResourceScheduler scheduler = resourceManager.getResourceScheduler(); - - List appsInA1 = scheduler.getAppsInQueue("a1"); - assertEquals(1, appsInA1.size()); - - List appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.contains(application_0.getApplicationAttemptId())); - assertTrue(appsInA.contains(application_1.getApplicationAttemptId())); - assertEquals(2, appsInA.size()); - - List appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(application_0.getApplicationAttemptId())); - assertTrue(appsInRoot.contains(application_1.getApplicationAttemptId())); - assertTrue(appsInRoot.contains(application_2.getApplicationAttemptId())); - assertEquals(3, appsInRoot.size()); - - Assert.assertNull(scheduler.getAppsInQueue("nonexistentqueue")); - } - - @Test - public void testAddAndRemoveAppFromCapacityScheduler() throws Exception { - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); - MockRM rm = new MockRM(conf); - @SuppressWarnings("unchecked") - AbstractYarnScheduler cs = - (AbstractYarnScheduler) rm - .getResourceScheduler(); - SchedulerApplication app = - TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( - cs.getSchedulerApplications(), cs, "a1"); - Assert.assertEquals("a1", app.getQueue().getQueueName()); - } - @Test public void testAsyncScheduling() throws Exception { Configuration conf = new Configuration(); @@ -1381,37 +1072,6 @@ public void run() { rm.stop(); } - @Test - public void testNumClusterNodes() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); - CapacityScheduler cs = new CapacityScheduler(); - cs.setConf(conf); - RMContext rmContext = TestUtils.getMockRMContext(); - cs.setRMContext(rmContext); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); - setupQueueConfiguration(csConf); - cs.init(csConf); - cs.start(); - assertEquals(0, cs.getNumClusterNodes()); - - RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1); - RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2); - cs.handle(new NodeAddedSchedulerEvent(n1)); - cs.handle(new NodeAddedSchedulerEvent(n2)); - assertEquals(2, cs.getNumClusterNodes()); - - cs.handle(new NodeRemovedSchedulerEvent(n1)); - assertEquals(1, cs.getNumClusterNodes()); - cs.handle(new NodeAddedSchedulerEvent(n1)); - assertEquals(2, cs.getNumClusterNodes()); - cs.handle(new NodeRemovedSchedulerEvent(n2)); - cs.handle(new NodeRemovedSchedulerEvent(n1)); - assertEquals(0, cs.getNumClusterNodes()); - - cs.stop(); - } - @Test(timeout = 120000) public void testPreemptionInfo() throws Exception { Configuration conf = new Configuration(); @@ -1553,1790 +1213,56 @@ public void testRecoverRequestAfterPreemption() throws Exception { Assert.assertTrue(containers.size() == 1); } - private MockRM setUpMove() { - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - return setUpMove(conf); - } - - private MockRM setUpMove(Configuration config) { - CapacitySchedulerConfiguration conf = - new CapacitySchedulerConfiguration(config); - setupQueueConfiguration(conf); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); - MockRM rm = new MockRM(conf); - rm.start(); - return rm; - } - @Test - public void testAppSubmission() throws Exception { + public void testPreemptionDisabled() throws Exception { + CapacityScheduler cs = new CapacityScheduler(); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); + RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new NMTokenSecretManagerInRM(conf), + new ClientToAMTokenSecretManagerInRM(), null); setupQueueConfiguration(conf); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); - conf.setQueues(A, new String[] {"a1", "a2", "b"}); - conf.setCapacity(A1, 20); - conf.setCapacity("root.a.b", 10); - MockRM rm = new MockRM(conf); - rm.start(); - - RMApp noParentQueueApp = submitAppAndWaitForState(rm, "q", RMAppState.FAILED); - Assert.assertEquals(RMAppState.FAILED, noParentQueueApp.getState()); - - RMApp ambiguousQueueApp = submitAppAndWaitForState(rm, "b", RMAppState.FAILED); - Assert.assertEquals(RMAppState.FAILED, ambiguousQueueApp.getState()); - - RMApp emptyPartQueueApp = submitAppAndWaitForState(rm, "root..a1", RMAppState.FAILED); - Assert.assertEquals(RMAppState.FAILED, emptyPartQueueApp.getState()); - - RMApp failedAutoQueue = submitAppAndWaitForState(rm, "root.a.b.c.d", RMAppState.FAILED); - Assert.assertEquals(RMAppState.FAILED, failedAutoQueue.getState()); - } - - private RMApp submitAppAndWaitForState(MockRM rm, String b, RMAppState state) throws Exception { - MockRMAppSubmissionData ambiguousQueueAppData = - MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) - .withWaitForAppAcceptedState(false) - .withAppName("app") - .withUser("user") - .withAcls(null) - .withQueue(b) - .withUnmanagedAM(false) - .build(); - RMApp app1 = MockRMAppSubmitter.submit(rm, ambiguousQueueAppData); - rm.waitForState(app1.getApplicationId(), state); - return app1; - } - - @Test - public void testMoveAppBasic() throws Exception { - MockRM rm = setUpMove(); - AbstractYarnScheduler scheduler = - (AbstractYarnScheduler) rm.getResourceScheduler(); - QueueMetrics metrics = scheduler.getRootQueueMetrics(); - Assert.assertEquals(0, metrics.getAppsPending()); - // submit an app - MockRMAppSubmissionData data = - MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) - .withAppName("test-move-1") - .withUser("user_0") - .withAcls(null) - .withQueue("a1") - .withUnmanagedAM(false) - .build(); - RMApp app = MockRMAppSubmitter.submit(rm, data); - ApplicationAttemptId appAttemptId = - rm.getApplicationReport(app.getApplicationId()) - .getCurrentApplicationAttemptId(); - // check preconditions - List appsInA1 = scheduler.getAppsInQueue("a1"); - assertEquals(1, appsInA1.size()); - String queue = - scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() - .getQueueName(); - Assert.assertEquals("a1", queue); - - List appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.contains(appAttemptId)); - assertEquals(1, appsInA.size()); - - List appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(1, appsInRoot.size()); - - assertEquals(1, metrics.getAppsPending()); - - List appsInB1 = scheduler.getAppsInQueue("b1"); - assertTrue(appsInB1.isEmpty()); - - List appsInB = scheduler.getAppsInQueue("b"); - assertTrue(appsInB.isEmpty()); - - // now move the app - scheduler.moveApplication(app.getApplicationId(), "b1"); - - // check postconditions - appsInB1 = scheduler.getAppsInQueue("b1"); - assertEquals(1, appsInB1.size()); - queue = - scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue() - .getQueueName(); - Assert.assertEquals("b1", queue); - - appsInB = scheduler.getAppsInQueue("b"); - assertTrue(appsInB.contains(appAttemptId)); - assertEquals(1, appsInB.size()); - - appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(1, appsInRoot.size()); - - assertEquals(1, metrics.getAppsPending()); - - appsInA1 = scheduler.getAppsInQueue("a1"); - assertTrue(appsInA1.isEmpty()); - - appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.isEmpty()); - - rm.stop(); - } - - @Test - public void testMoveAppPendingMetrics() throws Exception { - MockRM rm = setUpMove(); - AbstractYarnScheduler scheduler = - (AbstractYarnScheduler) rm.getResourceScheduler(); - QueueMetrics metrics = scheduler.getRootQueueMetrics(); - List appsInA1 = scheduler.getAppsInQueue("a1"); - List appsInB1 = scheduler.getAppsInQueue("b1"); - - assertEquals(0, appsInA1.size()); - assertEquals(0, appsInB1.size()); - Assert.assertEquals(0, metrics.getAppsPending()); - - // submit two apps in a1 - RMApp app1 = MockRMAppSubmitter.submit(rm, - MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) - .withAppName("test-move-1") - .withUser("user_0") - .withAcls(null) - .withQueue("a1") - .build()); - RMApp app2 = MockRMAppSubmitter.submit(rm, - MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) - .withAppName("test-move-2") - .withUser("user_0") - .withAcls(null) - .withQueue("a1") - .build()); - - appsInA1 = scheduler.getAppsInQueue("a1"); - appsInB1 = scheduler.getAppsInQueue("b1"); - assertEquals(2, appsInA1.size()); - assertEquals(0, appsInB1.size()); - assertEquals(2, metrics.getAppsPending()); - - // submit one app in b1 - RMApp app3 = MockRMAppSubmitter.submit(rm, - MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) - .withAppName("test-move-2") - .withUser("user_0") - .withAcls(null) - .withQueue("b1") - .build()); - - appsInA1 = scheduler.getAppsInQueue("a1"); - appsInB1 = scheduler.getAppsInQueue("b1"); - assertEquals(2, appsInA1.size()); - assertEquals(1, appsInB1.size()); - assertEquals(3, metrics.getAppsPending()); - - // now move the app1 from a1 to b1 - scheduler.moveApplication(app1.getApplicationId(), "b1"); - - appsInA1 = scheduler.getAppsInQueue("a1"); - appsInB1 = scheduler.getAppsInQueue("b1"); - assertEquals(1, appsInA1.size()); - assertEquals(2, appsInB1.size()); - assertEquals(3, metrics.getAppsPending()); - - // now move the app2 from a1 to b1 - scheduler.moveApplication(app2.getApplicationId(), "b1"); - - appsInA1 = scheduler.getAppsInQueue("a1"); - appsInB1 = scheduler.getAppsInQueue("b1"); - assertEquals(0, appsInA1.size()); - assertEquals(3, appsInB1.size()); - assertEquals(3, metrics.getAppsPending()); - - // now move the app3 from b1 to a1 - scheduler.moveApplication(app3.getApplicationId(), "a1"); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + cs.init(conf); + cs.start(); + cs.reinitialize(conf, rmContext); - appsInA1 = scheduler.getAppsInQueue("a1"); - appsInB1 = scheduler.getAppsInQueue("b1"); - assertEquals(1, appsInA1.size()); - assertEquals(2, appsInB1.size()); - assertEquals(3, metrics.getAppsPending()); - rm.stop(); - } + CSQueue rootQueue = cs.getRootQueue(); + CSQueue queueB = findQueue(rootQueue, B); + CSQueue queueB2 = findQueue(queueB, B2); - @Test - public void testMoveAppSameParent() throws Exception { - MockRM rm = setUpMove(); - AbstractYarnScheduler scheduler = - (AbstractYarnScheduler) rm.getResourceScheduler(); - - // submit an app - MockRMAppSubmissionData data = - MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) - .withAppName("test-move-1") - .withUser("user_0") - .withAcls(null) - .withQueue("a1") - .withUnmanagedAM(false) - .build(); - RMApp app = MockRMAppSubmitter.submit(rm, data); - ApplicationAttemptId appAttemptId = - rm.getApplicationReport(app.getApplicationId()) - .getCurrentApplicationAttemptId(); - - // check preconditions - List appsInA1 = scheduler.getAppsInQueue("a1"); - assertEquals(1, appsInA1.size()); - String queue = - scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() - .getQueueName(); - Assert.assertEquals("a1", queue); - - List appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.contains(appAttemptId)); - assertEquals(1, appsInA.size()); - - List appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(1, appsInRoot.size()); - - List appsInA2 = scheduler.getAppsInQueue("a2"); - assertTrue(appsInA2.isEmpty()); - - // now move the app - scheduler.moveApplication(app.getApplicationId(), "a2"); - - // check postconditions - appsInA2 = scheduler.getAppsInQueue("a2"); - assertEquals(1, appsInA2.size()); - queue = - scheduler.getApplicationAttempt(appsInA2.get(0)).getQueue() - .getQueueName(); - Assert.assertEquals("a2", queue); - - appsInA1 = scheduler.getAppsInQueue("a1"); - assertTrue(appsInA1.isEmpty()); - - appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.contains(appAttemptId)); - assertEquals(1, appsInA.size()); - - appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(1, appsInRoot.size()); - - rm.stop(); - } - - @Test - public void testMoveAppForMoveToQueueWithFreeCap() throws Exception { - - ResourceScheduler scheduler = resourceManager.getResourceScheduler(); - - NodeStatus mockNodeStatus = createMockNodeStatus(); - - // Register node1 - String host_0 = "host_0"; - NodeManager nm_0 = - registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, - Resources.createResource(4 * GB, 1), mockNodeStatus); - - // Register node2 - String host_1 = "host_1"; - NodeManager nm_1 = - registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, - Resources.createResource(2 * GB, 1), mockNodeStatus); - - // ResourceRequest priorities - Priority priority_0 = Priority.newInstance(0); - Priority priority_1 = Priority.newInstance(1); - - // Submit application_0 - Application application_0 = - new Application("user_0", "a1", resourceManager); - application_0.submit(); // app + app attempt event sent to scheduler - - application_0.addNodeManager(host_0, 1234, nm_0); - application_0.addNodeManager(host_1, 1234, nm_1); - - Resource capability_0_0 = Resources.createResource(1 * GB, 1); - application_0.addResourceRequestSpec(priority_1, capability_0_0); - - Resource capability_0_1 = Resources.createResource(2 * GB, 1); - application_0.addResourceRequestSpec(priority_0, capability_0_1); - - Task task_0_0 = - new Task(application_0, priority_1, new String[] { host_0, host_1 }); - application_0.addTask(task_0_0); - - // Submit application_1 - Application application_1 = - new Application("user_1", "b2", resourceManager); - application_1.submit(); // app + app attempt event sent to scheduler - - application_1.addNodeManager(host_0, 1234, nm_0); - application_1.addNodeManager(host_1, 1234, nm_1); - - Resource capability_1_0 = Resources.createResource(1 * GB, 1); - application_1.addResourceRequestSpec(priority_1, capability_1_0); - - Resource capability_1_1 = Resources.createResource(2 * GB, 1); - application_1.addResourceRequestSpec(priority_0, capability_1_1); - - Task task_1_0 = - new Task(application_1, priority_1, new String[] { host_0, host_1 }); - application_1.addTask(task_1_0); - - // Send resource requests to the scheduler - application_0.schedule(); // allocate - application_1.schedule(); // allocate - - // task_0_0 task_1_0 allocated, used=2G - nodeUpdate(nm_0); - - // nothing allocated - nodeUpdate(nm_1); - - // Get allocations from the scheduler - application_0.schedule(); // task_0_0 - checkApplicationResourceUsage(1 * GB, application_0); - - application_1.schedule(); // task_1_0 - checkApplicationResourceUsage(1 * GB, application_1); - - checkNodeResourceUsage(2 * GB, nm_0); // task_0_0 (1G) and task_1_0 (1G) 2G - // available - checkNodeResourceUsage(0 * GB, nm_1); // no tasks, 2G available - - // move app from a1(30% cap of total 10.5% cap) to b1(79,2% cap of 89,5% - // total cap) - scheduler.moveApplication(application_0.getApplicationId(), "b1"); - - // 2GB 1C - Task task_1_1 = - new Task(application_1, priority_0, - new String[] { ResourceRequest.ANY }); - application_1.addTask(task_1_1); - - application_1.schedule(); - - // 2GB 1C - Task task_0_1 = - new Task(application_0, priority_0, new String[] { host_0, host_1 }); - application_0.addTask(task_0_1); - - application_0.schedule(); - - // prev 2G used free 2G - nodeUpdate(nm_0); - - // prev 0G used free 2G - nodeUpdate(nm_1); - - // Get allocations from the scheduler - application_1.schedule(); - checkApplicationResourceUsage(3 * GB, application_1); - - // Get allocations from the scheduler - application_0.schedule(); - checkApplicationResourceUsage(3 * GB, application_0); - - checkNodeResourceUsage(4 * GB, nm_0); - checkNodeResourceUsage(2 * GB, nm_1); - - } - - @Test - public void testMoveAppSuccess() throws Exception { - - ResourceScheduler scheduler = resourceManager.getResourceScheduler(); - - NodeStatus mockNodeStatus = createMockNodeStatus(); - - // Register node1 - String host_0 = "host_0"; - NodeManager nm_0 = - registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, - Resources.createResource(5 * GB, 1), mockNodeStatus); - - // Register node2 - String host_1 = "host_1"; - NodeManager nm_1 = - registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, - Resources.createResource(5 * GB, 1), mockNodeStatus); - - // ResourceRequest priorities - Priority priority_0 = Priority.newInstance(0); - Priority priority_1 = Priority.newInstance(1); - - // Submit application_0 - Application application_0 = - new Application("user_0", "a1", resourceManager); - application_0.submit(); // app + app attempt event sent to scheduler - - application_0.addNodeManager(host_0, 1234, nm_0); - application_0.addNodeManager(host_1, 1234, nm_1); - - Resource capability_0_0 = Resources.createResource(3 * GB, 1); - application_0.addResourceRequestSpec(priority_1, capability_0_0); - - Resource capability_0_1 = Resources.createResource(2 * GB, 1); - application_0.addResourceRequestSpec(priority_0, capability_0_1); - - Task task_0_0 = - new Task(application_0, priority_1, new String[] { host_0, host_1 }); - application_0.addTask(task_0_0); - - // Submit application_1 - Application application_1 = - new Application("user_1", "b2", resourceManager); - application_1.submit(); // app + app attempt event sent to scheduler - - application_1.addNodeManager(host_0, 1234, nm_0); - application_1.addNodeManager(host_1, 1234, nm_1); - - Resource capability_1_0 = Resources.createResource(1 * GB, 1); - application_1.addResourceRequestSpec(priority_1, capability_1_0); - - Resource capability_1_1 = Resources.createResource(2 * GB, 1); - application_1.addResourceRequestSpec(priority_0, capability_1_1); - - Task task_1_0 = - new Task(application_1, priority_1, new String[] { host_0, host_1 }); - application_1.addTask(task_1_0); - - // Send resource requests to the scheduler - application_0.schedule(); // allocate - application_1.schedule(); // allocate - - // b2 can only run 1 app at a time - scheduler.moveApplication(application_0.getApplicationId(), "b2"); - - nodeUpdate(nm_0); - - nodeUpdate(nm_1); - - // Get allocations from the scheduler - application_0.schedule(); // task_0_0 - checkApplicationResourceUsage(0 * GB, application_0); - - application_1.schedule(); // task_1_0 - checkApplicationResourceUsage(1 * GB, application_1); - - // task_1_0 (1G) application_0 moved to b2 with max running app 1 so it is - // not scheduled - checkNodeResourceUsage(1 * GB, nm_0); - checkNodeResourceUsage(0 * GB, nm_1); - - // lets move application_0 to a queue where it can run - scheduler.moveApplication(application_0.getApplicationId(), "a2"); - application_0.schedule(); - - nodeUpdate(nm_1); - - // Get allocations from the scheduler - application_0.schedule(); // task_0_0 - checkApplicationResourceUsage(3 * GB, application_0); - - checkNodeResourceUsage(1 * GB, nm_0); - checkNodeResourceUsage(3 * GB, nm_1); - - } - - @Test(expected = YarnException.class) - public void testMoveAppViolateQueueState() throws Exception { - resourceManager = new ResourceManager() { - @Override - protected RMNodeLabelsManager createNodeLabelManager() { - RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); - mgr.init(getConfig()); - return mgr; - } - }; - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); - setupQueueConfiguration(csConf); - StringBuilder qState = new StringBuilder(); - qState.append(CapacitySchedulerConfiguration.PREFIX).append(B) - .append(CapacitySchedulerConfiguration.DOT) - .append(CapacitySchedulerConfiguration.STATE); - csConf.set(qState.toString(), QueueState.STOPPED.name()); - YarnConfiguration conf = new YarnConfiguration(csConf); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); - resourceManager.init(conf); - resourceManager.getRMContext().getContainerTokenSecretManager() - .rollMasterKey(); - resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); - ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start(); - mockContext = mock(RMContext.class); - when(mockContext.getConfigurationProvider()).thenReturn( - new LocalConfigurationProvider()); - - ResourceScheduler scheduler = resourceManager.getResourceScheduler(); - - NodeStatus mockNodeStatus = createMockNodeStatus(); - - // Register node1 - String host_0 = "host_0"; - NodeManager nm_0 = - registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, - Resources.createResource(6 * GB, 1), mockNodeStatus); - - // ResourceRequest priorities - Priority priority_0 = Priority.newInstance(0); - Priority priority_1 = Priority.newInstance(1); - - // Submit application_0 - Application application_0 = - new Application("user_0", "a1", resourceManager); - application_0.submit(); // app + app attempt event sent to scheduler - - application_0.addNodeManager(host_0, 1234, nm_0); - - Resource capability_0_0 = Resources.createResource(3 * GB, 1); - application_0.addResourceRequestSpec(priority_1, capability_0_0); - - Resource capability_0_1 = Resources.createResource(2 * GB, 1); - application_0.addResourceRequestSpec(priority_0, capability_0_1); - - Task task_0_0 = - new Task(application_0, priority_1, new String[] { host_0 }); - application_0.addTask(task_0_0); - - // Send resource requests to the scheduler - application_0.schedule(); // allocate - - // task_0_0 allocated - nodeUpdate(nm_0); - - // Get allocations from the scheduler - application_0.schedule(); // task_0_0 - checkApplicationResourceUsage(3 * GB, application_0); - - checkNodeResourceUsage(3 * GB, nm_0); - // b2 queue contains 3GB consumption app, - // add another 3GB will hit max capacity limit on queue b - scheduler.moveApplication(application_0.getApplicationId(), "b1"); - - } - - @Test - public void testMoveAppQueueMetricsCheck() throws Exception { - ResourceScheduler scheduler = resourceManager.getResourceScheduler(); - - NodeStatus mockNodeStatus = createMockNodeStatus(); - - // Register node1 - String host_0 = "host_0"; - NodeManager nm_0 = - registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, - Resources.createResource(5 * GB, 1), mockNodeStatus); - - // Register node2 - String host_1 = "host_1"; - NodeManager nm_1 = - registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, - Resources.createResource(5 * GB, 1), mockNodeStatus); - - // ResourceRequest priorities - Priority priority_0 = Priority.newInstance(0); - Priority priority_1 = Priority.newInstance(1); - - // Submit application_0 - Application application_0 = - new Application("user_0", "a1", resourceManager); - application_0.submit(); // app + app attempt event sent to scheduler - - application_0.addNodeManager(host_0, 1234, nm_0); - application_0.addNodeManager(host_1, 1234, nm_1); - - Resource capability_0_0 = Resources.createResource(3 * GB, 1); - application_0.addResourceRequestSpec(priority_1, capability_0_0); - - Resource capability_0_1 = Resources.createResource(2 * GB, 1); - application_0.addResourceRequestSpec(priority_0, capability_0_1); - - Task task_0_0 = - new Task(application_0, priority_1, new String[] { host_0, host_1 }); - application_0.addTask(task_0_0); - - // Submit application_1 - Application application_1 = - new Application("user_1", "b2", resourceManager); - application_1.submit(); // app + app attempt event sent to scheduler - - application_1.addNodeManager(host_0, 1234, nm_0); - application_1.addNodeManager(host_1, 1234, nm_1); - - Resource capability_1_0 = Resources.createResource(1 * GB, 1); - application_1.addResourceRequestSpec(priority_1, capability_1_0); - - Resource capability_1_1 = Resources.createResource(2 * GB, 1); - application_1.addResourceRequestSpec(priority_0, capability_1_1); - - Task task_1_0 = - new Task(application_1, priority_1, new String[] { host_0, host_1 }); - application_1.addTask(task_1_0); - - // Send resource requests to the scheduler - application_0.schedule(); // allocate - application_1.schedule(); // allocate - - nodeUpdate(nm_0); - - nodeUpdate(nm_1); - - CapacityScheduler cs = - (CapacityScheduler) resourceManager.getResourceScheduler(); - CSQueue origRootQ = cs.getRootQueue(); - CapacitySchedulerInfo oldInfo = - new CapacitySchedulerInfo(origRootQ, cs); - int origNumAppsA = getNumAppsInQueue("a", origRootQ.getChildQueues()); - int origNumAppsRoot = origRootQ.getNumApplications(); - - scheduler.moveApplication(application_0.getApplicationId(), "a2"); - - CSQueue newRootQ = cs.getRootQueue(); - int newNumAppsA = getNumAppsInQueue("a", newRootQ.getChildQueues()); - int newNumAppsRoot = newRootQ.getNumApplications(); - CapacitySchedulerInfo newInfo = - new CapacitySchedulerInfo(newRootQ, cs); - CapacitySchedulerLeafQueueInfo origOldA1 = - (CapacitySchedulerLeafQueueInfo) getQueueInfo("a1", oldInfo.getQueues()); - CapacitySchedulerLeafQueueInfo origNewA1 = - (CapacitySchedulerLeafQueueInfo) getQueueInfo("a1", newInfo.getQueues()); - CapacitySchedulerLeafQueueInfo targetOldA2 = - (CapacitySchedulerLeafQueueInfo) getQueueInfo("a2", oldInfo.getQueues()); - CapacitySchedulerLeafQueueInfo targetNewA2 = - (CapacitySchedulerLeafQueueInfo) getQueueInfo("a2", newInfo.getQueues()); - // originally submitted here - assertEquals(1, origOldA1.getNumApplications()); - assertEquals(1, origNumAppsA); - assertEquals(2, origNumAppsRoot); - // after the move - assertEquals(0, origNewA1.getNumApplications()); - assertEquals(1, newNumAppsA); - assertEquals(2, newNumAppsRoot); - // original consumption on a1 - assertEquals(3 * GB, origOldA1.getResourcesUsed().getMemorySize()); - assertEquals(1, origOldA1.getResourcesUsed().getvCores()); - assertEquals(0, origNewA1.getResourcesUsed().getMemorySize()); // after the move - assertEquals(0, origNewA1.getResourcesUsed().getvCores()); // after the move - // app moved here with live containers - assertEquals(3 * GB, targetNewA2.getResourcesUsed().getMemorySize()); - assertEquals(1, targetNewA2.getResourcesUsed().getvCores()); - // it was empty before the move - assertEquals(0, targetOldA2.getNumApplications()); - assertEquals(0, targetOldA2.getResourcesUsed().getMemorySize()); - assertEquals(0, targetOldA2.getResourcesUsed().getvCores()); - // after the app moved here - assertEquals(1, targetNewA2.getNumApplications()); - // 1 container on original queue before move - assertEquals(1, origOldA1.getNumContainers()); - // after the move the resource released - assertEquals(0, origNewA1.getNumContainers()); - // and moved to the new queue - assertEquals(1, targetNewA2.getNumContainers()); - // which originally didn't have any - assertEquals(0, targetOldA2.getNumContainers()); - // 1 user with 3GB - assertEquals(3 * GB, origOldA1.getUsers().getUsersList().get(0) - .getResourcesUsed().getMemorySize()); - // 1 user with 1 core - assertEquals(1, origOldA1.getUsers().getUsersList().get(0) - .getResourcesUsed().getvCores()); - // user ha no more running app in the orig queue - assertEquals(0, origNewA1.getUsers().getUsersList().size()); - // 1 user with 3GB - assertEquals(3 * GB, targetNewA2.getUsers().getUsersList().get(0) - .getResourcesUsed().getMemorySize()); - // 1 user with 1 core - assertEquals(1, targetNewA2.getUsers().getUsersList().get(0) - .getResourcesUsed().getvCores()); - - // Get allocations from the scheduler - application_0.schedule(); // task_0_0 - checkApplicationResourceUsage(3 * GB, application_0); - - application_1.schedule(); // task_1_0 - checkApplicationResourceUsage(1 * GB, application_1); - - // task_1_0 (1G) application_0 moved to b2 with max running app 1 so it is - // not scheduled - checkNodeResourceUsage(4 * GB, nm_0); - checkNodeResourceUsage(0 * GB, nm_1); - - } - - private int getNumAppsInQueue(String name, List queues) { - for (CSQueue queue : queues) { - if (queue.getQueueShortName().equals(name)) { - return queue.getNumApplications(); - } - } - return -1; - } - - private CapacitySchedulerQueueInfo getQueueInfo(String name, - CapacitySchedulerQueueInfoList info) { - if (info != null) { - for (CapacitySchedulerQueueInfo queueInfo : info.getQueueInfoList()) { - if (queueInfo.getQueueName().equals(name)) { - return queueInfo; - } else { - CapacitySchedulerQueueInfo result = - getQueueInfo(name, queueInfo.getQueues()); - if (result == null) { - continue; - } - return result; - } - } - } - return null; - } - - @Test - public void testMoveAllApps() throws Exception { - MockRM rm = setUpMove(); - AbstractYarnScheduler scheduler = - (AbstractYarnScheduler) rm.getResourceScheduler(); - - // submit an app - MockRMAppSubmissionData data = - MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) - .withAppName("test-move-1") - .withUser("user_0") - .withAcls(null) - .withQueue("a1") - .withUnmanagedAM(false) - .build(); - RMApp app = MockRMAppSubmitter.submit(rm, data); - ApplicationAttemptId appAttemptId = - rm.getApplicationReport(app.getApplicationId()) - .getCurrentApplicationAttemptId(); - - // check preconditions - List appsInA1 = scheduler.getAppsInQueue("a1"); - assertEquals(1, appsInA1.size()); - - List appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.contains(appAttemptId)); - assertEquals(1, appsInA.size()); - String queue = - scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() - .getQueueName(); - Assert.assertEquals("a1", queue); - - List appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(1, appsInRoot.size()); - - List appsInB1 = scheduler.getAppsInQueue("b1"); - assertTrue(appsInB1.isEmpty()); - - List appsInB = scheduler.getAppsInQueue("b"); - assertTrue(appsInB.isEmpty()); - - // now move the app - scheduler.moveAllApps("a1", "b1"); - - // check postconditions - Thread.sleep(1000); - appsInB1 = scheduler.getAppsInQueue("b1"); - assertEquals(1, appsInB1.size()); - queue = - scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue() - .getQueueName(); - Assert.assertEquals("b1", queue); - - appsInB = scheduler.getAppsInQueue("b"); - assertTrue(appsInB.contains(appAttemptId)); - assertEquals(1, appsInB.size()); - - appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(1, appsInRoot.size()); - - appsInA1 = scheduler.getAppsInQueue("a1"); - assertTrue(appsInA1.isEmpty()); - - appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.isEmpty()); - - rm.stop(); - } - - @Test - public void testMoveAllAppsInvalidDestination() throws Exception { - MockRM rm = setUpMove(); - YarnScheduler scheduler = rm.getResourceScheduler(); - - // submit an app - MockRMAppSubmissionData data = - MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) - .withAppName("test-move-1") - .withUser("user_0") - .withAcls(null) - .withQueue("a1") - .withUnmanagedAM(false) - .build(); - RMApp app = MockRMAppSubmitter.submit(rm, data); - ApplicationAttemptId appAttemptId = - rm.getApplicationReport(app.getApplicationId()) - .getCurrentApplicationAttemptId(); - - // check preconditions - List appsInA1 = scheduler.getAppsInQueue("a1"); - assertEquals(1, appsInA1.size()); - - List appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.contains(appAttemptId)); - assertEquals(1, appsInA.size()); - - List appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(1, appsInRoot.size()); - - List appsInB1 = scheduler.getAppsInQueue("b1"); - assertTrue(appsInB1.isEmpty()); - - List appsInB = scheduler.getAppsInQueue("b"); - assertTrue(appsInB.isEmpty()); - - // now move the app - try { - scheduler.moveAllApps("a1", "DOES_NOT_EXIST"); - Assert.fail(); - } catch (YarnException e) { - // expected - } - - // check postconditions, app should still be in a1 - appsInA1 = scheduler.getAppsInQueue("a1"); - assertEquals(1, appsInA1.size()); - - appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.contains(appAttemptId)); - assertEquals(1, appsInA.size()); - - appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(1, appsInRoot.size()); - - appsInB1 = scheduler.getAppsInQueue("b1"); - assertTrue(appsInB1.isEmpty()); - - appsInB = scheduler.getAppsInQueue("b"); - assertTrue(appsInB.isEmpty()); - - rm.stop(); - } - - @Test - public void testMoveAllAppsInvalidSource() throws Exception { - MockRM rm = setUpMove(); - YarnScheduler scheduler = rm.getResourceScheduler(); - - // submit an app - MockRMAppSubmissionData data = - MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) - .withAppName("test-move-1") - .withUser("user_0") - .withAcls(null) - .withQueue("a1") - .withUnmanagedAM(false) - .build(); - RMApp app = MockRMAppSubmitter.submit(rm, data); - ApplicationAttemptId appAttemptId = - rm.getApplicationReport(app.getApplicationId()) - .getCurrentApplicationAttemptId(); - - // check preconditions - List appsInA1 = scheduler.getAppsInQueue("a1"); - assertEquals(1, appsInA1.size()); - - List appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.contains(appAttemptId)); - assertEquals(1, appsInA.size()); - - List appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(1, appsInRoot.size()); - - List appsInB1 = scheduler.getAppsInQueue("b1"); - assertTrue(appsInB1.isEmpty()); - - List appsInB = scheduler.getAppsInQueue("b"); - assertTrue(appsInB.isEmpty()); - - // now move the app - try { - scheduler.moveAllApps("DOES_NOT_EXIST", "b1"); - Assert.fail(); - } catch (YarnException e) { - // expected - } - - // check postconditions, app should still be in a1 - appsInA1 = scheduler.getAppsInQueue("a1"); - assertEquals(1, appsInA1.size()); - - appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.contains(appAttemptId)); - assertEquals(1, appsInA.size()); - - appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(1, appsInRoot.size()); - - appsInB1 = scheduler.getAppsInQueue("b1"); - assertTrue(appsInB1.isEmpty()); - - appsInB = scheduler.getAppsInQueue("b"); - assertTrue(appsInB.isEmpty()); - - rm.stop(); - } - - @Test(timeout = 60000) - public void testMoveAttemptNotAdded() throws Exception { - Configuration conf = new Configuration(); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); - MockRM rm = new MockRM(getCapacityConfiguration(conf)); - rm.start(); - CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); - - ApplicationId appId = BuilderUtils.newApplicationId(100, 1); - ApplicationAttemptId appAttemptId = - BuilderUtils.newApplicationAttemptId(appId, 1); - - RMAppAttemptMetrics attemptMetric = - new RMAppAttemptMetrics(appAttemptId, rm.getRMContext()); - RMAppImpl app = mock(RMAppImpl.class); - when(app.getApplicationId()).thenReturn(appId); - RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class); - Container container = mock(Container.class); - when(attempt.getMasterContainer()).thenReturn(container); - ApplicationSubmissionContext submissionContext = - mock(ApplicationSubmissionContext.class); - when(attempt.getSubmissionContext()).thenReturn(submissionContext); - when(attempt.getAppAttemptId()).thenReturn(appAttemptId); - when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric); - when(app.getCurrentAppAttempt()).thenReturn(attempt); - - rm.getRMContext().getRMApps().put(appId, app); - - SchedulerEvent addAppEvent = - new AppAddedSchedulerEvent(appId, "a1", "user"); - try { - cs.moveApplication(appId, "b1"); - fail("Move should throw exception app not available"); - } catch (YarnException e) { - assertEquals("App to be moved application_100_0001 not found.", - e.getMessage()); - } - cs.handle(addAppEvent); - cs.moveApplication(appId, "b1"); - SchedulerEvent addAttemptEvent = - new AppAttemptAddedSchedulerEvent(appAttemptId, false); - cs.handle(addAttemptEvent); - CSQueue rootQ = cs.getRootQueue(); - CSQueue queueB = cs.getQueue("b"); - CSQueue queueA = cs.getQueue("a"); - CSQueue queueA1 = cs.getQueue("a1"); - CSQueue queueB1 = cs.getQueue("b1"); - Assert.assertEquals(1, rootQ.getNumApplications()); - Assert.assertEquals(0, queueA.getNumApplications()); - Assert.assertEquals(1, queueB.getNumApplications()); - Assert.assertEquals(0, queueA1.getNumApplications()); - Assert.assertEquals(1, queueB1.getNumApplications()); - - rm.close(); - } - - @Test - public void testRemoveAttemptMoveAdded() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - CapacityScheduler.class); - conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); - // Create Mock RM - MockRM rm = new MockRM(getCapacityConfiguration(conf)); - CapacityScheduler sch = (CapacityScheduler) rm.getResourceScheduler(); - // add node - Resource newResource = Resource.newInstance(4 * GB, 1); - RMNode node = MockNodes.newNodeInfo(0, newResource, 1, "127.0.0.1"); - SchedulerEvent addNode = new NodeAddedSchedulerEvent(node); - sch.handle(addNode); - - ApplicationAttemptId appAttemptId = appHelper(rm, sch, 100, 1, "a1", "user"); - - // get Queues - CSQueue queueA1 = sch.getQueue("a1"); - CSQueue queueB = sch.getQueue("b"); - CSQueue queueB1 = sch.getQueue("b1"); - - // add Running rm container and simulate live containers to a1 - ContainerId newContainerId = ContainerId.newContainerId(appAttemptId, 2); - RMContainerImpl rmContainer = mock(RMContainerImpl.class); - when(rmContainer.getState()).thenReturn(RMContainerState.RUNNING); - Container container2 = mock(Container.class); - when(rmContainer.getContainer()).thenReturn(container2); - Resource resource = Resource.newInstance(1024, 1); - when(container2.getResource()).thenReturn(resource); - when(rmContainer.getExecutionType()).thenReturn(ExecutionType.GUARANTEED); - when(container2.getNodeId()).thenReturn(node.getNodeID()); - when(container2.getId()).thenReturn(newContainerId); - when(rmContainer.getNodeLabelExpression()) - .thenReturn(RMNodeLabelsManager.NO_LABEL); - when(rmContainer.getContainerId()).thenReturn(newContainerId); - sch.getApplicationAttempt(appAttemptId).getLiveContainersMap() - .put(newContainerId, rmContainer); - QueueMetrics queueA1M = queueA1.getMetrics(); - queueA1M.incrPendingResources(rmContainer.getNodeLabelExpression(), - "user1", 1, resource); - queueA1M.allocateResources(rmContainer.getNodeLabelExpression(), - "user1", resource); - // remove attempt - sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId, - RMAppAttemptState.KILLED, true)); - // Move application to queue b1 - sch.moveApplication(appAttemptId.getApplicationId(), "b1"); - // Check queue metrics after move - Assert.assertEquals(0, queueA1.getNumApplications()); - Assert.assertEquals(1, queueB.getNumApplications()); - Assert.assertEquals(0, queueB1.getNumApplications()); - - // Release attempt add event - ApplicationAttemptId appAttemptId2 = - BuilderUtils.newApplicationAttemptId(appAttemptId.getApplicationId(), 2); - SchedulerEvent addAttemptEvent2 = - new AppAttemptAddedSchedulerEvent(appAttemptId2, true); - sch.handle(addAttemptEvent2); - - // Check metrics after attempt added - Assert.assertEquals(0, queueA1.getNumApplications()); - Assert.assertEquals(1, queueB.getNumApplications()); - Assert.assertEquals(1, queueB1.getNumApplications()); - - - QueueMetrics queueB1M = queueB1.getMetrics(); - QueueMetrics queueBM = queueB.getMetrics(); - // Verify allocation MB of current state - Assert.assertEquals(0, queueA1M.getAllocatedMB()); - Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores()); - Assert.assertEquals(1024, queueB1M.getAllocatedMB()); - Assert.assertEquals(1, queueB1M.getAllocatedVirtualCores()); - - // remove attempt - sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId2, - RMAppAttemptState.FINISHED, false)); - - Assert.assertEquals(0, queueA1M.getAllocatedMB()); - Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores()); - Assert.assertEquals(0, queueB1M.getAllocatedMB()); - Assert.assertEquals(0, queueB1M.getAllocatedVirtualCores()); - - verifyQueueMetrics(queueB1M); - verifyQueueMetrics(queueBM); - // Verify queue A1 metrics - verifyQueueMetrics(queueA1M); - rm.close(); - } - - private void verifyQueueMetrics(QueueMetrics queue) { - Assert.assertEquals(0, queue.getPendingMB()); - Assert.assertEquals(0, queue.getActiveUsers()); - Assert.assertEquals(0, queue.getActiveApps()); - Assert.assertEquals(0, queue.getAppsPending()); - Assert.assertEquals(0, queue.getAppsRunning()); - Assert.assertEquals(0, queue.getAllocatedMB()); - Assert.assertEquals(0, queue.getAllocatedVirtualCores()); - } - - private Configuration getCapacityConfiguration(Configuration config) { - CapacitySchedulerConfiguration conf = - new CapacitySchedulerConfiguration(config); - - // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] {"a", "b"}); - conf.setCapacity(A, 50); - conf.setCapacity(B, 50); - conf.setQueues(A, new String[] {"a1", "a2"}); - conf.setCapacity(A1, 50); - conf.setCapacity(A2, 50); - conf.setQueues(B, new String[] {"b1"}); - conf.setCapacity(B1, 100); - return conf; - } - - @Test - public void testKillAllAppsInQueue() throws Exception { - MockRM rm = setUpMove(); - AbstractYarnScheduler scheduler = - (AbstractYarnScheduler) rm.getResourceScheduler(); - - // submit an app - MockRMAppSubmissionData data = - MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) - .withAppName("test-move-1") - .withUser("user_0") - .withAcls(null) - .withQueue("a1") - .withUnmanagedAM(false) - .build(); - RMApp app = MockRMAppSubmitter.submit(rm, data); - ApplicationAttemptId appAttemptId = - rm.getApplicationReport(app.getApplicationId()) - .getCurrentApplicationAttemptId(); - - // check preconditions - List appsInA1 = scheduler.getAppsInQueue("a1"); - assertEquals(1, appsInA1.size()); - - List appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.contains(appAttemptId)); - assertEquals(1, appsInA.size()); - String queue = - scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() - .getQueueName(); - Assert.assertEquals("a1", queue); - - List appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(1, appsInRoot.size()); - - // now kill the app - scheduler.killAllAppsInQueue("a1"); - - // check postconditions - rm.waitForState(app.getApplicationId(), RMAppState.KILLED); - rm.waitForAppRemovedFromScheduler(app.getApplicationId()); - appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.isEmpty()); - - appsInA1 = scheduler.getAppsInQueue("a1"); - assertTrue(appsInA1.isEmpty()); - - appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.isEmpty()); - - rm.stop(); - } - - @Test - public void testKillAllAppsInvalidSource() throws Exception { - MockRM rm = setUpMove(); - YarnScheduler scheduler = rm.getResourceScheduler(); - - // submit an app - MockRMAppSubmissionData data = - MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) - .withAppName("test-move-1") - .withUser("user_0") - .withAcls(null) - .withQueue("a1") - .withUnmanagedAM(false) - .build(); - RMApp app = MockRMAppSubmitter.submit(rm, data); - ApplicationAttemptId appAttemptId = - rm.getApplicationReport(app.getApplicationId()) - .getCurrentApplicationAttemptId(); - - // check preconditions - List appsInA1 = scheduler.getAppsInQueue("a1"); - assertEquals(1, appsInA1.size()); - - List appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.contains(appAttemptId)); - assertEquals(1, appsInA.size()); - - List appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(1, appsInRoot.size()); - - // now kill the app - try { - scheduler.killAllAppsInQueue("DOES_NOT_EXIST"); - Assert.fail(); - } catch (YarnException e) { - // expected - } - - // check postconditions, app should still be in a1 - appsInA1 = scheduler.getAppsInQueue("a1"); - assertEquals(1, appsInA1.size()); - - appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.contains(appAttemptId)); - assertEquals(1, appsInA.size()); - - appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(1, appsInRoot.size()); - - rm.stop(); - } - - // Test to ensure that we don't carry out reservation on nodes - // that have no CPU available when using the DominantResourceCalculator - @Test(timeout = 30000) - public void testAppReservationWithDominantResourceCalculator() throws Exception { - CapacitySchedulerConfiguration csconf = - new CapacitySchedulerConfiguration(); - csconf.setResourceComparator(DominantResourceCalculator.class); - - YarnConfiguration conf = new YarnConfiguration(csconf); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); - - MockRM rm = new MockRM(conf); - rm.start(); - - MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10 * GB, 1); - - // register extra nodes to bump up cluster resource - MockNM nm2 = rm.registerNode("127.0.0.1:1235", 10 * GB, 4); - rm.registerNode("127.0.0.1:1236", 10 * GB, 4); - - RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm); - // kick the scheduling - nm1.nodeHeartbeat(true); - RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); - MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); - am1.registerAppAttempt(); - SchedulerNodeReport report_nm1 = - rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - - // check node report - Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(9 * GB, report_nm1.getAvailableResource().getMemorySize()); - - // add request for containers - am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 1 * GB, 1, 1); - am1.schedule(); // send the request - - // kick the scheduler, container reservation should not happen - nm1.nodeHeartbeat(true); - Thread.sleep(1000); - AllocateResponse allocResponse = am1.schedule(); - ApplicationResourceUsageReport report = - rm.getResourceScheduler().getAppResourceUsageReport( - attempt1.getAppAttemptId()); - Assert.assertEquals(0, allocResponse.getAllocatedContainers().size()); - Assert.assertEquals(0, report.getNumReservedContainers()); - - // container should get allocated on this node - nm2.nodeHeartbeat(true); - - while (allocResponse.getAllocatedContainers().size() == 0) { - Thread.sleep(100); - allocResponse = am1.schedule(); - } - report = - rm.getResourceScheduler().getAppResourceUsageReport( - attempt1.getAppAttemptId()); - Assert.assertEquals(1, allocResponse.getAllocatedContainers().size()); - Assert.assertEquals(0, report.getNumReservedContainers()); - rm.stop(); - } - - @Test - public void testPreemptionDisabled() throws Exception { - CapacityScheduler cs = new CapacityScheduler(); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); - RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null); - setupQueueConfiguration(conf); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - cs.init(conf); - cs.start(); - cs.reinitialize(conf, rmContext); - - CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueB = findQueue(rootQueue, B); - CSQueue queueB2 = findQueue(queueB, B2); - - // When preemption turned on for the whole system - // (yarn.resourcemanager.scheduler.monitor.enable=true), and with no other - // preemption properties set, queue root.b.b2 should be preemptable. - assertFalse("queue " + B2 + " should default to preemptable", - queueB2.getPreemptionDisabled()); + // When preemption turned on for the whole system + // (yarn.resourcemanager.scheduler.monitor.enable=true), and with no other + // preemption properties set, queue root.b.b2 should be preemptable. + assertFalse("queue " + B2 + " should default to preemptable", + queueB2.getPreemptionDisabled()); // Disable preemption at the root queue level. // The preemption property should be inherited from root all the - // way down so that root.b.b2 should NOT be preemptable. - conf.setPreemptionDisabled(rootQueue.getQueuePath(), true); - cs.reinitialize(conf, rmContext); - assertTrue( - "queue " + B2 + " should have inherited non-preemptability from root", - queueB2.getPreemptionDisabled()); - - // Enable preemption for root (grandparent) but disable for root.b (parent). - // root.b.b2 should inherit property from parent and NOT be preemptable - conf.setPreemptionDisabled(rootQueue.getQueuePath(), false); - conf.setPreemptionDisabled(queueB.getQueuePath(), true); - cs.reinitialize(conf, rmContext); - assertTrue( - "queue " + B2 + " should have inherited non-preemptability from parent", - queueB2.getPreemptionDisabled()); - - // When preemption is turned on for root.b.b2, it should be preemptable - // even though preemption is disabled on root.b (parent). - conf.setPreemptionDisabled(queueB2.getQueuePath(), false); - cs.reinitialize(conf, rmContext); - assertFalse("queue " + B2 + " should have been preemptable", - queueB2.getPreemptionDisabled()); - } - - @Test - public void testRefreshQueuesMaxAllocationRefresh() throws Exception { - // queue refresh should not allow changing the maximum allocation setting - // per queue to be smaller than previous setting - CapacityScheduler cs = new CapacityScheduler(); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - cs.init(conf); - cs.start(); - cs.reinitialize(conf, mockContext); - checkQueueStructureCapacities(cs); - - assertEquals("max allocation in CS", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - cs.getMaximumResourceCapability().getMemorySize()); - assertEquals("max allocation for A1", - Resources.none(), - conf.getQueueMaximumAllocation(A1)); - assertEquals("max allocation", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize()); - - CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueA = findQueue(rootQueue, A); - CSQueue queueA1 = findQueue(queueA, A1); - assertEquals("queue max allocation", ((LeafQueue) queueA1) - .getMaximumAllocation().getMemorySize(), 8192); - - setMaxAllocMb(conf, A1, 4096); - - try { - cs.reinitialize(conf, mockContext); - fail("should have thrown exception"); - } catch (IOException e) { - assertTrue("max allocation exception", - e.getCause().toString().contains("not be decreased")); - } - - setMaxAllocMb(conf, A1, 8192); - cs.reinitialize(conf, mockContext); - - setMaxAllocVcores(conf, A1, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES - 1); - try { - cs.reinitialize(conf, mockContext); - fail("should have thrown exception"); - } catch (IOException e) { - assertTrue("max allocation exception", - e.getCause().toString().contains("not be decreased")); - } - } - - @Test - public void testRefreshQueuesMaxAllocationPerQueueLarge() throws Exception { - // verify we can't set the allocation per queue larger then cluster setting - CapacityScheduler cs = new CapacityScheduler(); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); - cs.init(conf); - cs.start(); - // change max allocation for B3 queue to be larger then cluster max - setMaxAllocMb(conf, B3, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 2048); - try { - cs.reinitialize(conf, mockContext); - fail("should have thrown exception"); - } catch (IOException e) { - assertTrue("maximum allocation exception", - e.getCause().getMessage().contains("maximum allocation")); - } - - setMaxAllocMb(conf, B3, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); - cs.reinitialize(conf, mockContext); - - setMaxAllocVcores(conf, B3, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1); - try { - cs.reinitialize(conf, mockContext); - fail("should have thrown exception"); - } catch (IOException e) { - assertTrue("maximum allocation exception", - e.getCause().getMessage().contains("maximum allocation")); - } - } - - @Test - public void testRefreshQueuesMaxAllocationRefreshLarger() throws Exception { - // queue refresh should allow max allocation per queue to go larger - CapacityScheduler cs = new CapacityScheduler(); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); - setMaxAllocMb(conf, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); - setMaxAllocVcores(conf, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); - setMaxAllocMb(conf, A1, 4096); - setMaxAllocVcores(conf, A1, 2); - cs.init(conf); - cs.start(); - cs.reinitialize(conf, mockContext); - checkQueueStructureCapacities(cs); - - CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueA = findQueue(rootQueue, A); - CSQueue queueA1 = findQueue(queueA, A1); - - assertEquals("max capability MB in CS", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - cs.getMaximumResourceCapability().getMemorySize()); - assertEquals("max capability vcores in CS", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - cs.getMaximumResourceCapability().getVirtualCores()); - assertEquals("max allocation MB A1", - 4096, - queueA1.getMaximumAllocation().getMemorySize()); - assertEquals("max allocation vcores A1", - 2, - queueA1.getMaximumAllocation().getVirtualCores()); - assertEquals("cluster max allocation MB", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize()); - assertEquals("cluster max allocation vcores", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - ResourceUtils.fetchMaximumAllocationFromConfig(conf).getVirtualCores()); - - assertEquals("queue max allocation", 4096, - queueA1.getMaximumAllocation().getMemorySize()); - - setMaxAllocMb(conf, A1, 6144); - setMaxAllocVcores(conf, A1, 3); - cs.reinitialize(conf, null); - // conf will have changed but we shouldn't be able to change max allocation - // for the actual queue - assertEquals("max allocation MB A1", 6144, - queueA1.getMaximumAllocation().getMemorySize()); - assertEquals("max allocation vcores A1", 3, - queueA1.getMaximumAllocation().getVirtualCores()); - assertEquals("max allocation MB cluster", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize()); - assertEquals("max allocation vcores cluster", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - ResourceUtils.fetchMaximumAllocationFromConfig(conf).getVirtualCores()); - assertEquals("queue max allocation MB", 6144, - queueA1.getMaximumAllocation().getMemorySize()); - assertEquals("queue max allocation vcores", 3, - queueA1.getMaximumAllocation().getVirtualCores()); - assertEquals("max capability MB cluster", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - cs.getMaximumResourceCapability().getMemorySize()); - assertEquals("cluster max capability vcores", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - cs.getMaximumResourceCapability().getVirtualCores()); - } - - @Test - public void testRefreshQueuesMaxAllocationCSError() throws Exception { - // Try to refresh the cluster level max allocation size to be smaller - // and it should error out - CapacityScheduler cs = new CapacityScheduler(); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); - setMaxAllocMb(conf, 10240); - setMaxAllocVcores(conf, 10); - setMaxAllocMb(conf, A1, 4096); - setMaxAllocVcores(conf, A1, 4); - cs.init(conf); - cs.start(); - cs.reinitialize(conf, mockContext); - checkQueueStructureCapacities(cs); - - assertEquals("max allocation MB in CS", 10240, - cs.getMaximumResourceCapability().getMemorySize()); - assertEquals("max allocation vcores in CS", 10, - cs.getMaximumResourceCapability().getVirtualCores()); - - setMaxAllocMb(conf, 6144); - try { - cs.reinitialize(conf, mockContext); - fail("should have thrown exception"); - } catch (IOException e) { - assertTrue("max allocation exception", - e.getCause().toString().contains("not be decreased")); - } - - setMaxAllocMb(conf, 10240); - cs.reinitialize(conf, mockContext); - - setMaxAllocVcores(conf, 8); - try { - cs.reinitialize(conf, mockContext); - fail("should have thrown exception"); - } catch (IOException e) { - assertTrue("max allocation exception", - e.getCause().toString().contains("not be decreased")); - } - } - - @Test - public void testRefreshQueuesMaxAllocationCSLarger() throws Exception { - // Try to refresh the cluster level max allocation size to be larger - // and verify that if there is no setting per queue it uses the - // cluster level setting. - CapacityScheduler cs = new CapacityScheduler(); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); - setMaxAllocMb(conf, 10240); - setMaxAllocVcores(conf, 10); - setMaxAllocMb(conf, A1, 4096); - setMaxAllocVcores(conf, A1, 4); - cs.init(conf); - cs.start(); - cs.reinitialize(conf, mockContext); - checkQueueStructureCapacities(cs); - - assertEquals("max allocation MB in CS", 10240, - cs.getMaximumResourceCapability().getMemorySize()); - assertEquals("max allocation vcores in CS", 10, - cs.getMaximumResourceCapability().getVirtualCores()); - - CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueA = findQueue(rootQueue, A); - CSQueue queueB = findQueue(rootQueue, B); - CSQueue queueA1 = findQueue(queueA, A1); - CSQueue queueA2 = findQueue(queueA, A2); - CSQueue queueB2 = findQueue(queueB, B2); - - assertEquals("queue A1 max allocation MB", 4096, - queueA1.getMaximumAllocation().getMemorySize()); - assertEquals("queue A1 max allocation vcores", 4, - queueA1.getMaximumAllocation().getVirtualCores()); - assertEquals("queue A2 max allocation MB", 10240, - queueA2.getMaximumAllocation().getMemorySize()); - assertEquals("queue A2 max allocation vcores", 10, - queueA2.getMaximumAllocation().getVirtualCores()); - assertEquals("queue B2 max allocation MB", 10240, - queueB2.getMaximumAllocation().getMemorySize()); - assertEquals("queue B2 max allocation vcores", 10, - queueB2.getMaximumAllocation().getVirtualCores()); - - setMaxAllocMb(conf, 12288); - setMaxAllocVcores(conf, 12); - cs.reinitialize(conf, null); - // cluster level setting should change and any queues without - // per queue setting - assertEquals("max allocation MB in CS", 12288, - cs.getMaximumResourceCapability().getMemorySize()); - assertEquals("max allocation vcores in CS", 12, - cs.getMaximumResourceCapability().getVirtualCores()); - assertEquals("queue A1 max MB allocation", 4096, - queueA1.getMaximumAllocation().getMemorySize()); - assertEquals("queue A1 max vcores allocation", 4, - queueA1.getMaximumAllocation().getVirtualCores()); - assertEquals("queue A2 max MB allocation", 12288, - queueA2.getMaximumAllocation().getMemorySize()); - assertEquals("queue A2 max vcores allocation", 12, - queueA2.getMaximumAllocation().getVirtualCores()); - assertEquals("queue B2 max MB allocation", 12288, - queueB2.getMaximumAllocation().getMemorySize()); - assertEquals("queue B2 max vcores allocation", 12, - queueB2.getMaximumAllocation().getVirtualCores()); - } - - @Test - public void testQueuesMaxAllocationInheritance() throws Exception { - // queue level max allocation is set by the queue configuration explicitly - // or inherits from the parent. - - CapacityScheduler cs = new CapacityScheduler(); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); - setMaxAllocMb(conf, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); - setMaxAllocVcores(conf, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); - - // Test the child queue overrides - setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT, - "memory-mb=4096,vcores=2"); - setMaxAllocation(conf, A1, "memory-mb=6144,vcores=2"); - setMaxAllocation(conf, B, "memory-mb=5120, vcores=2"); - setMaxAllocation(conf, B2, "memory-mb=1024, vcores=2"); - - cs.init(conf); - cs.start(); - cs.reinitialize(conf, mockContext); - checkQueueStructureCapacities(cs); - - CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueA = findQueue(rootQueue, A); - CSQueue queueB = findQueue(rootQueue, B); - CSQueue queueA1 = findQueue(queueA, A1); - CSQueue queueA2 = findQueue(queueA, A2); - CSQueue queueB1 = findQueue(queueB, B1); - CSQueue queueB2 = findQueue(queueB, B2); - - assertEquals("max capability MB in CS", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - cs.getMaximumResourceCapability().getMemorySize()); - assertEquals("max capability vcores in CS", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - cs.getMaximumResourceCapability().getVirtualCores()); - assertEquals("max allocation MB A1", - 6144, - queueA1.getMaximumAllocation().getMemorySize()); - assertEquals("max allocation vcores A1", - 2, - queueA1.getMaximumAllocation().getVirtualCores()); - assertEquals("max allocation MB A2", 4096, - queueA2.getMaximumAllocation().getMemorySize()); - assertEquals("max allocation vcores A2", - 2, - queueA2.getMaximumAllocation().getVirtualCores()); - assertEquals("max allocation MB B", 5120, - queueB.getMaximumAllocation().getMemorySize()); - assertEquals("max allocation MB B1", 5120, - queueB1.getMaximumAllocation().getMemorySize()); - assertEquals("max allocation MB B2", 1024, - queueB2.getMaximumAllocation().getMemorySize()); - - // Test get the max-allocation from different parent - unsetMaxAllocation(conf, A1); - unsetMaxAllocation(conf, B); - unsetMaxAllocation(conf, B1); - setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT, - "memory-mb=6144,vcores=2"); - setMaxAllocation(conf, A, "memory-mb=8192,vcores=2"); - - cs.reinitialize(conf, mockContext); - - assertEquals("max capability MB in CS", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - cs.getMaximumResourceCapability().getMemorySize()); - assertEquals("max capability vcores in CS", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - cs.getMaximumResourceCapability().getVirtualCores()); - assertEquals("max allocation MB A1", - 8192, - queueA1.getMaximumAllocation().getMemorySize()); - assertEquals("max allocation vcores A1", - 2, - queueA1.getMaximumAllocation().getVirtualCores()); - assertEquals("max allocation MB B1", - 6144, - queueB1.getMaximumAllocation().getMemorySize()); - assertEquals("max allocation vcores B1", - 2, - queueB1.getMaximumAllocation().getVirtualCores()); - - // Test the default - unsetMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT); - unsetMaxAllocation(conf, A); - unsetMaxAllocation(conf, A1); - cs.reinitialize(conf, mockContext); - - assertEquals("max capability MB in CS", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - cs.getMaximumResourceCapability().getMemorySize()); - assertEquals("max capability vcores in CS", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - cs.getMaximumResourceCapability().getVirtualCores()); - assertEquals("max allocation MB A1", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - queueA1.getMaximumAllocation().getMemorySize()); - assertEquals("max allocation vcores A1", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - queueA1.getMaximumAllocation().getVirtualCores()); - assertEquals("max allocation MB A2", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - queueA2.getMaximumAllocation().getMemorySize()); - assertEquals("max allocation vcores A2", - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - queueA2.getMaximumAllocation().getVirtualCores()); - } - - @Test - public void testVerifyQueuesMaxAllocationConf() throws Exception { - // queue level max allocation can't exceed the cluster setting - - CapacityScheduler cs = new CapacityScheduler(); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); - setMaxAllocMb(conf, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); - setMaxAllocVcores(conf, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); - - long largerMem = - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 1024; - long largerVcores = - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES+10; - - cs.init(conf); - cs.start(); - cs.reinitialize(conf, mockContext); - checkQueueStructureCapacities(cs); - - setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT, - "memory-mb=" + largerMem + ",vcores=2"); - try { - cs.reinitialize(conf, mockContext); - fail("Queue Root maximum allocation can't exceed the cluster setting"); - } catch(Exception e) { - assertTrue("maximum allocation exception", - e.getCause().getMessage().contains("maximum allocation")); - } + // way down so that root.b.b2 should NOT be preemptable. + conf.setPreemptionDisabled(rootQueue.getQueuePath(), true); + cs.reinitialize(conf, rmContext); + assertTrue( + "queue " + B2 + " should have inherited non-preemptability from root", + queueB2.getPreemptionDisabled()); - setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT, - "memory-mb=4096,vcores=2"); - setMaxAllocation(conf, A, "memory-mb=6144,vcores=2"); - setMaxAllocation(conf, A1, "memory-mb=" + largerMem + ",vcores=2"); - try { - cs.reinitialize(conf, mockContext); - fail("Queue A1 maximum allocation can't exceed the cluster setting"); - } catch(Exception e) { - assertTrue("maximum allocation exception", - e.getCause().getMessage().contains("maximum allocation")); - } - setMaxAllocation(conf, A1, "memory-mb=8192" + ",vcores=" + largerVcores); - try { - cs.reinitialize(conf, mockContext); - fail("Queue A1 maximum allocation can't exceed the cluster setting"); - } catch(Exception e) { - assertTrue("maximum allocation exception", - e.getCause().getMessage().contains("maximum allocation")); - } + // Enable preemption for root (grandparent) but disable for root.b (parent). + // root.b.b2 should inherit property from parent and NOT be preemptable + conf.setPreemptionDisabled(rootQueue.getQueuePath(), false); + conf.setPreemptionDisabled(queueB.getQueuePath(), true); + cs.reinitialize(conf, rmContext); + assertTrue( + "queue " + B2 + " should have inherited non-preemptability from parent", + queueB2.getPreemptionDisabled()); + // When preemption is turned on for root.b.b2, it should be preemptable + // even though preemption is disabled on root.b (parent). + conf.setPreemptionDisabled(queueB2.getQueuePath(), false); + cs.reinitialize(conf, rmContext); + assertFalse("queue " + B2 + " should have been preemptable", + queueB2.getPreemptionDisabled()); } private void waitContainerAllocated(MockAM am, int mem, int nContainer, @@ -3995,35 +1921,6 @@ public void testHeadRoomCalculationWithDRC() throws Exception { assertEquals(15, fiCaApp2.getHeadroom().getVirtualCores()); } - @Test - public void testDefaultNodeLabelExpressionQueueConfig() throws Exception { - CapacityScheduler cs = new CapacityScheduler(); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); - conf.setDefaultNodeLabelExpression("root.a", " x"); - conf.setDefaultNodeLabelExpression("root.b", " y "); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - cs.init(conf); - cs.start(); - - QueueInfo queueInfoA = cs.getQueueInfo("a", true, false); - Assert.assertEquals("Queue Name should be a", "a", - queueInfoA.getQueueName()); - Assert.assertEquals("Queue Path should be root.a", "root.a", - queueInfoA.getQueuePath()); - Assert.assertEquals("Default Node Label Expression should be x", "x", - queueInfoA.getDefaultNodeLabelExpression()); - - QueueInfo queueInfoB = cs.getQueueInfo("b", true, false); - Assert.assertEquals("Queue Name should be b", "b", - queueInfoB.getQueueName()); - Assert.assertEquals("Queue Path should be root.b", "root.b", - queueInfoB.getQueuePath()); - Assert.assertEquals("Default Node Label Expression should be y", "y", - queueInfoB.getDefaultNodeLabelExpression()); - } - @Test(timeout = 60000) public void testAMLimitUsage() throws Exception { @@ -4216,44 +2113,6 @@ public Boolean get() { rm.stop(); } - private void setMaxAllocMb(Configuration conf, int maxAllocMb) { - conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - maxAllocMb); - } - - private void setMaxAllocMb(CapacitySchedulerConfiguration conf, - String queueName, int maxAllocMb) { - String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) - + MAXIMUM_ALLOCATION_MB; - conf.setInt(propName, maxAllocMb); - } - - private void setMaxAllocVcores(Configuration conf, int maxAllocVcores) { - conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - maxAllocVcores); - } - - private void setMaxAllocVcores(CapacitySchedulerConfiguration conf, - String queueName, int maxAllocVcores) { - String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) - + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES; - conf.setInt(propName, maxAllocVcores); - } - - private void setMaxAllocation(CapacitySchedulerConfiguration conf, - String queueName, String maxAllocation) { - String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) - + MAXIMUM_ALLOCATION; - conf.set(propName, maxAllocation); - } - - private void unsetMaxAllocation(CapacitySchedulerConfiguration conf, - String queueName) { - String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) - + MAXIMUM_ALLOCATION; - conf.unset(propName); - } - private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); RMContainer rmContainer = cs.getRMContainer(containerId); @@ -4264,171 +2123,6 @@ private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) { Assert.fail("Cannot find RMContainer"); } } - @Test - public void testRemovedNodeDecomissioningNode() throws Exception { - NodeStatus mockNodeStatus = createMockNodeStatus(); - - // Register nodemanager - NodeManager nm = registerNode("host_decom", 1234, 2345, - NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4), - mockNodeStatus); - - RMNode node = - resourceManager.getRMContext().getRMNodes().get(nm.getNodeId()); - // Send a heartbeat to kick the tires on the Scheduler - NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); - resourceManager.getResourceScheduler().handle(nodeUpdate); - - // force remove the node to simulate race condition - ((CapacityScheduler) resourceManager.getResourceScheduler()).getNodeTracker(). - removeNode(nm.getNodeId()); - // Kick off another heartbeat with the node state mocked to decommissioning - RMNode spyNode = - Mockito.spy(resourceManager.getRMContext().getRMNodes() - .get(nm.getNodeId())); - when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING); - resourceManager.getResourceScheduler().handle( - new NodeUpdateSchedulerEvent(spyNode)); - } - - @Test - public void testResourceUpdateDecommissioningNode() throws Exception { - // Mock the RMNodeResourceUpdate event handler to update SchedulerNode - // to have 0 available resource - RMContext spyContext = Mockito.spy(resourceManager.getRMContext()); - Dispatcher mockDispatcher = mock(AsyncDispatcher.class); - when(mockDispatcher.getEventHandler()).thenReturn(new EventHandler() { - @Override - public void handle(Event event) { - if (event instanceof RMNodeResourceUpdateEvent) { - RMNodeResourceUpdateEvent resourceEvent = - (RMNodeResourceUpdateEvent) event; - resourceManager - .getResourceScheduler() - .getSchedulerNode(resourceEvent.getNodeId()) - .updateTotalResource(resourceEvent.getResourceOption().getResource()); - } - } - }); - Mockito.doReturn(mockDispatcher).when(spyContext).getDispatcher(); - ((CapacityScheduler) resourceManager.getResourceScheduler()) - .setRMContext(spyContext); - ((AsyncDispatcher) mockDispatcher).start(); - - NodeStatus mockNodeStatus = createMockNodeStatus(); - - // Register node - String host_0 = "host_0"; - NodeManager nm_0 = registerNode(host_0, 1234, 2345, - NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4), - mockNodeStatus); - // ResourceRequest priorities - Priority priority_0 = Priority.newInstance(0); - - // Submit an application - Application application_0 = - new Application("user_0", "a1", resourceManager); - application_0.submit(); - - application_0.addNodeManager(host_0, 1234, nm_0); - - Resource capability_0_0 = Resources.createResource(1 * GB, 1); - application_0.addResourceRequestSpec(priority_0, capability_0_0); - - Task task_0_0 = - new Task(application_0, priority_0, new String[] { host_0 }); - application_0.addTask(task_0_0); - - // Send resource requests to the scheduler - application_0.schedule(); - - nodeUpdate(nm_0); - // Kick off another heartbeat with the node state mocked to decommissioning - // This should update the schedulernodes to have 0 available resource - RMNode spyNode = - Mockito.spy(resourceManager.getRMContext().getRMNodes() - .get(nm_0.getNodeId())); - when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING); - resourceManager.getResourceScheduler().handle( - new NodeUpdateSchedulerEvent(spyNode)); - - // Get allocations from the scheduler - application_0.schedule(); - - // Check the used resource is 1 GB 1 core - Assert.assertEquals(1 * GB, nm_0.getUsed().getMemorySize()); - Resource usedResource = - resourceManager.getResourceScheduler() - .getSchedulerNode(nm_0.getNodeId()).getAllocatedResource(); - Assert.assertEquals("Used Resource Memory Size should be 1GB", 1 * GB, - usedResource.getMemorySize()); - Assert.assertEquals("Used Resource Virtual Cores should be 1", 1, - usedResource.getVirtualCores()); - // Check total resource of scheduler node is also changed to 1 GB 1 core - Resource totalResource = - resourceManager.getResourceScheduler() - .getSchedulerNode(nm_0.getNodeId()).getTotalResource(); - Assert.assertEquals("Total Resource Memory Size should be 1GB", 1 * GB, - totalResource.getMemorySize()); - Assert.assertEquals("Total Resource Virtual Cores should be 1", 1, - totalResource.getVirtualCores()); - // Check the available resource is 0/0 - Resource availableResource = - resourceManager.getResourceScheduler() - .getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource(); - Assert.assertEquals("Available Resource Memory Size should be 0", 0, - availableResource.getMemorySize()); - Assert.assertEquals("Available Resource Memory Size should be 0", 0, - availableResource.getVirtualCores()); - // Kick off another heartbeat where the RMNodeResourceUpdateEvent would - // be skipped for DECOMMISSIONING state since the total resource is - // already equal to used resource from the previous heartbeat. - when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING); - resourceManager.getResourceScheduler().handle( - new NodeUpdateSchedulerEvent(spyNode)); - verify(mockDispatcher, times(4)).getEventHandler(); - } - - @Test - public void testSchedulingOnRemovedNode() throws Exception { - Configuration conf = new YarnConfiguration(); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); - conf.setBoolean( - CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, - false); - - MockRM rm = new MockRM(conf); - rm.start(); - RMApp app = MockRMAppSubmitter.submitWithMemory(100, rm); - rm.drainEvents(); - - MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10240, 10); - MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); - - //remove nm2 to keep am alive - MockNM nm2 = rm.registerNode("127.0.0.1:1235", 10240, 10); - - am.allocate(ResourceRequest.ANY, 2048, 1, null); - - CapacityScheduler scheduler = - (CapacityScheduler) rm.getRMContext().getScheduler(); - FiCaSchedulerNode node = - (FiCaSchedulerNode) - scheduler.getNodeTracker().getNode(nm2.getNodeId()); - scheduler.handle(new NodeRemovedSchedulerEvent( - rm.getRMContext().getRMNodes().get(nm2.getNodeId()))); - // schedulerNode is removed, try allocate a container - scheduler.allocateContainersToNode(new SimpleCandidateNodeSet<>(node), - true); - - AppAttemptRemovedSchedulerEvent appRemovedEvent1 = - new AppAttemptRemovedSchedulerEvent( - am.getApplicationAttemptId(), - RMAppAttemptState.FINISHED, false); - scheduler.handle(appRemovedEvent1); - rm.stop(); - } @Test public void testCSReservationWithRootUnblocked() throws Exception { @@ -4630,37 +2324,6 @@ public void testCSQueueBlocked() throws Exception { rm.stop(); } - private ApplicationAttemptId appHelper(MockRM rm, CapacityScheduler cs, - int clusterTs, int appId, String queue, - String user) { - ApplicationId appId1 = BuilderUtils.newApplicationId(clusterTs, appId); - ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId( - appId1, appId); - - RMAppAttemptMetrics attemptMetric1 = - new RMAppAttemptMetrics(appAttemptId1, rm.getRMContext()); - RMAppImpl app1 = mock(RMAppImpl.class); - when(app1.getApplicationId()).thenReturn(appId1); - RMAppAttemptImpl attempt1 = mock(RMAppAttemptImpl.class); - Container container = mock(Container.class); - when(attempt1.getMasterContainer()).thenReturn(container); - ApplicationSubmissionContext submissionContext = mock( - ApplicationSubmissionContext.class); - when(attempt1.getSubmissionContext()).thenReturn(submissionContext); - when(attempt1.getAppAttemptId()).thenReturn(appAttemptId1); - when(attempt1.getRMAppAttemptMetrics()).thenReturn(attemptMetric1); - when(app1.getCurrentAppAttempt()).thenReturn(attempt1); - rm.getRMContext().getRMApps().put(appId1, app1); - - SchedulerEvent addAppEvent1 = - new AppAddedSchedulerEvent(appId1, queue, user); - cs.handle(addAppEvent1); - SchedulerEvent addAttemptEvent1 = - new AppAttemptAddedSchedulerEvent(appAttemptId1, false); - cs.handle(addAttemptEvent1); - return appAttemptId1; - } - @Test public void testAppAttemptLocalityStatistics() throws Exception { Configuration conf = @@ -4722,256 +2385,6 @@ protected RMNodeLabelsManager createNodeLabelManager() { attemptMetrics.getLocalityStatistics()); } - /** - * Test for queue deletion. - * @throws Exception - */ - @Test - public void testRefreshQueuesWithQueueDelete() throws Exception { - CapacityScheduler cs = new CapacityScheduler(); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null); - setupQueueConfiguration(conf); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - cs.init(conf); - cs.start(); - cs.reinitialize(conf, rmContext); - checkQueueStructureCapacities(cs); - - // test delete leaf queue when there is application running. - Map queues = - cs.getCapacitySchedulerQueueManager().getShortNameQueues(); - String b1QTobeDeleted = "b1"; - LeafQueue csB1Queue = Mockito.spy((LeafQueue) queues.get(b1QTobeDeleted)); - when(csB1Queue.getState()).thenReturn(QueueState.DRAINING) - .thenReturn(QueueState.STOPPED); - cs.getCapacitySchedulerQueueManager().addQueue(b1QTobeDeleted, csB1Queue); - conf = new CapacitySchedulerConfiguration(); - setupQueueConfigurationWithoutB1(conf); - try { - cs.reinitialize(conf, mockContext); - fail("Expected to throw exception when refresh queue tries to delete a" - + " queue with running apps"); - } catch (IOException e) { - // ignore - } - - // test delete leaf queue(root.b.b1) when there is no application running. - conf = new CapacitySchedulerConfiguration(); - setupQueueConfigurationWithoutB1(conf); - try { - cs.reinitialize(conf, mockContext); - } catch (IOException e) { - LOG.error( - "Expected to NOT throw exception when refresh queue tries to delete" - + " a queue WITHOUT running apps", - e); - fail("Expected to NOT throw exception when refresh queue tries to delete" - + " a queue WITHOUT running apps"); - } - CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueB = findQueue(rootQueue, B); - CSQueue queueB3 = findQueue(queueB, B1); - assertNull("Refresh needs to support delete of leaf queue ", queueB3); - - // reset back to default configuration for testing parent queue delete - conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); - cs.reinitialize(conf, rmContext); - checkQueueStructureCapacities(cs); - - // set the configurations such that it fails once but should be successfull - // next time - queues = cs.getCapacitySchedulerQueueManager().getShortNameQueues(); - CSQueue bQueue = Mockito.spy((ParentQueue) queues.get("b")); - when(bQueue.getState()).thenReturn(QueueState.DRAINING) - .thenReturn(QueueState.STOPPED); - cs.getCapacitySchedulerQueueManager().addQueue("b", bQueue); - - bQueue = Mockito.spy((LeafQueue) queues.get("b1")); - when(bQueue.getState()).thenReturn(QueueState.STOPPED); - cs.getCapacitySchedulerQueueManager().addQueue("b1", bQueue); - - bQueue = Mockito.spy((LeafQueue) queues.get("b2")); - when(bQueue.getState()).thenReturn(QueueState.STOPPED); - cs.getCapacitySchedulerQueueManager().addQueue("b2", bQueue); - - bQueue = Mockito.spy((LeafQueue) queues.get("b3")); - when(bQueue.getState()).thenReturn(QueueState.STOPPED); - cs.getCapacitySchedulerQueueManager().addQueue("b3", bQueue); - - // test delete Parent queue when there is application running. - conf = new CapacitySchedulerConfiguration(); - setupQueueConfigurationWithoutB(conf); - try { - cs.reinitialize(conf, mockContext); - fail("Expected to throw exception when refresh queue tries to delete a" - + " parent queue with running apps in children queue"); - } catch (IOException e) { - // ignore - } - - // test delete Parent queue when there is no application running. - conf = new CapacitySchedulerConfiguration(); - setupQueueConfigurationWithoutB(conf); - try { - cs.reinitialize(conf, mockContext); - } catch (IOException e) { - fail("Expected to not throw exception when refresh queue tries to delete" - + " a queue without running apps"); - } - rootQueue = cs.getRootQueue(); - queueB = findQueue(rootQueue, B); - String message = - "Refresh needs to support delete of Parent queue and its children."; - assertNull(message, queueB); - assertNull(message, - cs.getCapacitySchedulerQueueManager().getQueues().get("b")); - assertNull(message, - cs.getCapacitySchedulerQueueManager().getQueues().get("b1")); - assertNull(message, - cs.getCapacitySchedulerQueueManager().getQueues().get("b2")); - - cs.stop(); - } - - /** - * Test for all child queue deletion and thus making parent queue a child. - * @throws Exception - */ - @Test - public void testRefreshQueuesWithAllChildQueuesDeleted() throws Exception { - CapacityScheduler cs = new CapacityScheduler(); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null); - setupQueueConfiguration(conf); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - cs.init(conf); - cs.start(); - cs.reinitialize(conf, rmContext); - checkQueueStructureCapacities(cs); - - // test delete all leaf queues when there is no application running. - Map queues = - cs.getCapacitySchedulerQueueManager().getShortNameQueues(); - - CSQueue bQueue = Mockito.spy((LeafQueue) queues.get("b1")); - when(bQueue.getState()).thenReturn(QueueState.RUNNING) - .thenReturn(QueueState.STOPPED); - cs.getCapacitySchedulerQueueManager().addQueue("b1", bQueue); - - bQueue = Mockito.spy((LeafQueue) queues.get("b2")); - when(bQueue.getState()).thenReturn(QueueState.STOPPED); - cs.getCapacitySchedulerQueueManager().addQueue("b2", bQueue); - - bQueue = Mockito.spy((LeafQueue) queues.get("b3")); - when(bQueue.getState()).thenReturn(QueueState.STOPPED); - cs.getCapacitySchedulerQueueManager().addQueue("b3", bQueue); - - conf = new CapacitySchedulerConfiguration(); - setupQueueConfWithoutChildrenOfB(conf); - - // test convert parent queue to leaf queue(root.b) when there is no - // application running. - try { - cs.reinitialize(conf, mockContext); - fail("Expected to throw exception when refresh queue tries to make parent" - + " queue a child queue when one of its children is still running."); - } catch (IOException e) { - //do not do anything, expected exception - } - - // test delete leaf queues(root.b.b1,b2,b3) when there is no application - // running. - try { - cs.reinitialize(conf, mockContext); - } catch (IOException e) { - e.printStackTrace(); - fail("Expected to NOT throw exception when refresh queue tries to delete" - + " all children of a parent queue(without running apps)."); - } - CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueB = findQueue(rootQueue, B); - assertNotNull("Parent Queue B should not be deleted", queueB); - Assert.assertTrue("As Queue'B children are not deleted", - queueB instanceof LeafQueue); - - String message = - "Refresh needs to support delete of all children of Parent queue."; - assertNull(message, - cs.getCapacitySchedulerQueueManager().getQueues().get("b3")); - assertNull(message, - cs.getCapacitySchedulerQueueManager().getQueues().get("b1")); - assertNull(message, - cs.getCapacitySchedulerQueueManager().getQueues().get("b2")); - - cs.stop(); - } - - /** - * Test if we can convert a leaf queue to a parent queue - * @throws Exception - */ - @Test (timeout = 10000) - public void testConvertLeafQueueToParentQueue() throws Exception { - CapacityScheduler cs = new CapacityScheduler(); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null); - setupQueueConfiguration(conf); - cs.setConf(new YarnConfiguration()); - cs.setRMContext(resourceManager.getRMContext()); - cs.init(conf); - cs.start(); - cs.reinitialize(conf, rmContext); - checkQueueStructureCapacities(cs); - - String targetQueue = "b1"; - CSQueue b1 = cs.getQueue(targetQueue); - Assert.assertEquals(QueueState.RUNNING, b1.getState()); - - // test if we can convert a leaf queue which is in RUNNING state - conf = new CapacitySchedulerConfiguration(); - setupQueueConfigurationWithB1AsParentQueue(conf); - try { - cs.reinitialize(conf, mockContext); - fail("Expected to throw exception when refresh queue tries to convert" - + " a child queue to a parent queue."); - } catch (IOException e) { - // ignore - } - - // now set queue state for b1 to STOPPED - conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); - conf.set("yarn.scheduler.capacity.root.b.b1.state", "STOPPED"); - cs.reinitialize(conf, mockContext); - Assert.assertEquals(QueueState.STOPPED, b1.getState()); - - // test if we can convert a leaf queue which is in STOPPED state - conf = new CapacitySchedulerConfiguration(); - setupQueueConfigurationWithB1AsParentQueue(conf); - try { - cs.reinitialize(conf, mockContext); - } catch (IOException e) { - fail("Expected to NOT throw exception when refresh queue tries" - + " to convert a leaf queue WITHOUT running apps"); - } - b1 = cs.getQueue(targetQueue); - Assert.assertTrue(b1 instanceof ParentQueue); - Assert.assertEquals(QueueState.RUNNING, b1.getState()); - Assert.assertTrue(!b1.getChildQueues().isEmpty()); - } @Test(timeout = 30000) public void testAMLimitDouble() throws Exception { @@ -5250,166 +2663,6 @@ public void testContainerAllocationLocalitySkipped() throws Exception { ContainerAllocation.QUEUE_SKIPPED.getAllocationState()); } - @Test - public void testMoveAppWithActiveUsersWithOnlyPendingApps() throws Exception { - - YarnConfiguration conf = new YarnConfiguration(); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); - - CapacitySchedulerConfiguration newConf = - new CapacitySchedulerConfiguration(conf); - - // Define top-level queues - newConf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] { "a", "b" }); - - newConf.setCapacity(A, 50); - newConf.setCapacity(B, 50); - - // Define 2nd-level queues - newConf.setQueues(A, new String[] { "a1" }); - newConf.setCapacity(A1, 100); - newConf.setUserLimitFactor(A1, 2.0f); - newConf.setMaximumAMResourcePercentPerPartition(A1, "", 0.1f); - - newConf.setQueues(B, new String[] { "b1" }); - newConf.setCapacity(B1, 100); - newConf.setUserLimitFactor(B1, 2.0f); - - LOG.info("Setup top-level queues a and b"); - - MockRM rm = new MockRM(newConf); - rm.start(); - - CapacityScheduler scheduler = - (CapacityScheduler) rm.getResourceScheduler(); - - MockNM nm1 = rm.registerNode("h1:1234", 16 * GB); - - // submit an app - MockRMAppSubmissionData data3 = - MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) - .withAppName("test-move-1") - .withUser("u1") - .withAcls(null) - .withQueue("a1") - .withUnmanagedAM(false) - .build(); - RMApp app = MockRMAppSubmitter.submit(rm, data3); - MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm1); - - ApplicationAttemptId appAttemptId = - rm.getApplicationReport(app.getApplicationId()) - .getCurrentApplicationAttemptId(); - - MockRMAppSubmissionData data2 = - MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm) - .withAppName("app") - .withUser("u2") - .withAcls(null) - .withQueue("a1") - .withUnmanagedAM(false) - .build(); - RMApp app2 = MockRMAppSubmitter.submit(rm, data2); - MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); - - MockRMAppSubmissionData data1 = - MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm) - .withAppName("app") - .withUser("u3") - .withAcls(null) - .withQueue("a1") - .withUnmanagedAM(false) - .build(); - RMApp app3 = MockRMAppSubmitter.submit(rm, data1); - - MockRMAppSubmissionData data = - MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm) - .withAppName("app") - .withUser("u4") - .withAcls(null) - .withQueue("a1") - .withUnmanagedAM(false) - .build(); - RMApp app4 = MockRMAppSubmitter.submit(rm, data); - - // Each application asks 50 * 1GB containers - am1.allocate("*", 1 * GB, 50, null); - am2.allocate("*", 1 * GB, 50, null); - - CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); - RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId()); - - // check preconditions - List appsInA1 = scheduler.getAppsInQueue("a1"); - assertEquals(4, appsInA1.size()); - String queue = - scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() - .getQueueName(); - Assert.assertEquals("a1", queue); - - List appsInA = scheduler.getAppsInQueue("a"); - assertTrue(appsInA.contains(appAttemptId)); - assertEquals(4, appsInA.size()); - - List appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(4, appsInRoot.size()); - - List appsInB1 = scheduler.getAppsInQueue("b1"); - assertTrue(appsInB1.isEmpty()); - - List appsInB = scheduler.getAppsInQueue("b"); - assertTrue(appsInB.isEmpty()); - - UsersManager um = - (UsersManager) scheduler.getQueue("a1").getAbstractUsersManager(); - - assertEquals(4, um.getNumActiveUsers()); - assertEquals(2, um.getNumActiveUsersWithOnlyPendingApps()); - - // now move the app - scheduler.moveAllApps("a1", "b1"); - - //Triggering this event so that user limit computation can - //happen again - for (int i = 0; i < 10; i++) { - cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); - Thread.sleep(500); - } - - // check postconditions - appsInB1 = scheduler.getAppsInQueue("b1"); - - assertEquals(4, appsInB1.size()); - queue = - scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue() - .getQueueName(); - Assert.assertEquals("b1", queue); - - appsInB = scheduler.getAppsInQueue("b"); - assertTrue(appsInB.contains(appAttemptId)); - assertEquals(4, appsInB.size()); - - appsInRoot = scheduler.getAppsInQueue("root"); - assertTrue(appsInRoot.contains(appAttemptId)); - assertEquals(4, appsInRoot.size()); - - List oldAppsInA1 = scheduler.getAppsInQueue("a1"); - assertEquals(0, oldAppsInA1.size()); - - UsersManager um_b1 = - (UsersManager) scheduler.getQueue("b1").getAbstractUsersManager(); - - assertEquals(2, um_b1.getNumActiveUsers()); - assertEquals(2, um_b1.getNumActiveUsersWithOnlyPendingApps()); - - appsInB1 = scheduler.getAppsInQueue("b1"); - assertEquals(4, appsInB1.size()); - rm.close(); - } - @Test public void testCSQueueMetrics() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java new file mode 100644 index 00000000000000..9943e03e4ad747 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java @@ -0,0 +1,1499 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.util.Lists; +import org.apache.hadoop.yarn.LocalConfigurationProvider; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.api.records.NodeStatus; +import org.apache.hadoop.yarn.server.resourcemanager.Application; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; +import org.apache.hadoop.yarn.server.resourcemanager.NodeManager; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.Task; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import static org.apache.hadoop.yarn.server.resourcemanager.MockNM.createMockNodeStatus; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.appHelper; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkApplicationResourceUsage; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkNodeResourceUsage; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createMockRMContext; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createResourceManager; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.nodeUpdate; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.registerNode; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.setUpMove; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.stopResourceManager; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestCapacitySchedulerApps { + + private ResourceManager resourceManager = null; + private RMContext mockContext; + + @Before + public void setUp() throws Exception { + resourceManager = createResourceManager(); + mockContext = createMockRMContext(); + } + + @After + public void tearDown() throws Exception { + stopResourceManager(resourceManager); + } + + @Test + public void testGetAppsInQueue() throws Exception { + Application application0 = new Application("user_0", "a1", resourceManager); + application0.submit(); + + Application application1 = new Application("user_0", "a2", resourceManager); + application1.submit(); + + Application application2 = new Application("user_0", "b2", resourceManager); + application2.submit(); + + ResourceScheduler scheduler = resourceManager.getResourceScheduler(); + + List appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + + List appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(application0.getApplicationAttemptId())); + assertTrue(appsInA.contains(application1.getApplicationAttemptId())); + assertEquals(2, appsInA.size()); + + List appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(application0.getApplicationAttemptId())); + assertTrue(appsInRoot.contains(application1.getApplicationAttemptId())); + assertTrue(appsInRoot.contains(application2.getApplicationAttemptId())); + assertEquals(3, appsInRoot.size()); + + Assert.assertNull(scheduler.getAppsInQueue("nonexistentqueue")); + } + + @Test + public void testAddAndRemoveAppFromCapacityScheduler() throws Exception { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + MockRM rm = new MockRM(conf); + @SuppressWarnings("unchecked") + AbstractYarnScheduler cs = + (AbstractYarnScheduler) rm + .getResourceScheduler(); + SchedulerApplication app = + TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( + cs.getSchedulerApplications(), cs, "a1"); + Assert.assertEquals("a1", app.getQueue().getQueueName()); + } + + @Test + public void testKillAllAppsInQueue() throws Exception { + MockRM rm = setUpMove(); + AbstractYarnScheduler scheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + + // submit an app + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) + .withAppName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanagedAM(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + // check preconditions + List appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + + List appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + String queue = + scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() + .getQueueName(); + Assert.assertEquals("a1", queue); + + List appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + // now kill the app + scheduler.killAllAppsInQueue("a1"); + + // check postconditions + rm.waitForState(app.getApplicationId(), RMAppState.KILLED); + rm.waitForAppRemovedFromScheduler(app.getApplicationId()); + appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.isEmpty()); + + appsInA1 = scheduler.getAppsInQueue("a1"); + assertTrue(appsInA1.isEmpty()); + + appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.isEmpty()); + + rm.stop(); + } + + @Test + public void testKillAllAppsInvalidSource() throws Exception { + MockRM rm = setUpMove(); + YarnScheduler scheduler = rm.getResourceScheduler(); + + // submit an app + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) + .withAppName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanagedAM(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + // check preconditions + List appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + + List appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + + List appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + // now kill the app + try { + scheduler.killAllAppsInQueue("DOES_NOT_EXIST"); + Assert.fail(); + } catch (YarnException e) { + // expected + } + + // check postconditions, app should still be in a1 + appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + + appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + + appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + rm.stop(); + } + + // Test to ensure that we don't carry out reservation on nodes + // that have no CPU available when using the DominantResourceCalculator + @Test(timeout = 30000) + public void testAppReservationWithDominantResourceCalculator() throws Exception { + CapacitySchedulerConfiguration csconf = + new CapacitySchedulerConfiguration(); + csconf.setResourceComparator(DominantResourceCalculator.class); + + YarnConfiguration conf = new YarnConfiguration(csconf); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + MockRM rm = new MockRM(conf); + rm.start(); + + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10 * GB, 1); + + // register extra nodes to bump up cluster resource + MockNM nm2 = rm.registerNode("127.0.0.1:1235", 10 * GB, 4); + rm.registerNode("127.0.0.1:1236", 10 * GB, 4); + + RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm); + // kick the scheduling + nm1.nodeHeartbeat(true); + RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); + MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); + am1.registerAppAttempt(); + SchedulerNodeReport reportNm1 = + rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); + + // check node report + Assert.assertEquals(1 * GB, reportNm1.getUsedResource().getMemorySize()); + Assert.assertEquals(9 * GB, reportNm1.getAvailableResource().getMemorySize()); + + // add request for containers + am1.addRequests(new String[]{"127.0.0.1", "127.0.0.2"}, 1 * GB, 1, 1); + am1.schedule(); // send the request + + // kick the scheduler, container reservation should not happen + nm1.nodeHeartbeat(true); + Thread.sleep(1000); + AllocateResponse allocResponse = am1.schedule(); + ApplicationResourceUsageReport report = + rm.getResourceScheduler().getAppResourceUsageReport( + attempt1.getAppAttemptId()); + Assert.assertEquals(0, allocResponse.getAllocatedContainers().size()); + Assert.assertEquals(0, report.getNumReservedContainers()); + + // container should get allocated on this node + nm2.nodeHeartbeat(true); + + while (allocResponse.getAllocatedContainers().size() == 0) { + Thread.sleep(100); + allocResponse = am1.schedule(); + } + report = + rm.getResourceScheduler().getAppResourceUsageReport( + attempt1.getAppAttemptId()); + Assert.assertEquals(1, allocResponse.getAllocatedContainers().size()); + Assert.assertEquals(0, report.getNumReservedContainers()); + rm.stop(); + } + + @Test + public void testMoveAppBasic() throws Exception { + MockRM rm = setUpMove(); + AbstractYarnScheduler scheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + QueueMetrics metrics = scheduler.getRootQueueMetrics(); + Assert.assertEquals(0, metrics.getAppsPending()); + // submit an app + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) + .withAppName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanagedAM(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + // check preconditions + List appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + String queue = + scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() + .getQueueName(); + Assert.assertEquals("a1", queue); + + List appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + + List appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + assertEquals(1, metrics.getAppsPending()); + + List appsInB1 = scheduler.getAppsInQueue("b1"); + assertTrue(appsInB1.isEmpty()); + + List appsInB = scheduler.getAppsInQueue("b"); + assertTrue(appsInB.isEmpty()); + + // now move the app + scheduler.moveApplication(app.getApplicationId(), "b1"); + + // check postconditions + appsInB1 = scheduler.getAppsInQueue("b1"); + assertEquals(1, appsInB1.size()); + queue = + scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue() + .getQueueName(); + Assert.assertEquals("b1", queue); + + appsInB = scheduler.getAppsInQueue("b"); + assertTrue(appsInB.contains(appAttemptId)); + assertEquals(1, appsInB.size()); + + appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + assertEquals(1, metrics.getAppsPending()); + + appsInA1 = scheduler.getAppsInQueue("a1"); + assertTrue(appsInA1.isEmpty()); + + appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.isEmpty()); + + rm.stop(); + } + + @Test + public void testMoveAppPendingMetrics() throws Exception { + MockRM rm = setUpMove(); + ResourceScheduler scheduler = rm.getResourceScheduler(); + assertApps(scheduler, 0, 0, 0); + + // submit two apps in a1 + RMApp app1 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) + .withAppName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .build()); + RMApp app2 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) + .withAppName("test-move-2") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .build()); + assertApps(scheduler, 2, 0, 2); + + // submit one app in b1 + RMApp app3 = MockRMAppSubmitter.submit(rm, + MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) + .withAppName("test-move-2") + .withUser("user_0") + .withAcls(null) + .withQueue("b1") + .build()); + assertApps(scheduler, 2, 1, 3); + + // now move the app1 from a1 to b1 + scheduler.moveApplication(app1.getApplicationId(), "b1"); + assertApps(scheduler, 1, 2, 3); + + // now move the app2 from a1 to b1 + scheduler.moveApplication(app2.getApplicationId(), "b1"); + assertApps(scheduler, 0, 3, 3); + + // now move the app3 from b1 to a1 + scheduler.moveApplication(app3.getApplicationId(), "a1"); + assertApps(scheduler, 1, 2, 3); + rm.stop(); + } + + private void assertApps(ResourceScheduler scheduler, + int a1Size, + int b1Size, + int appsPending) { + assertAppsSize(scheduler, "a1", a1Size); + assertAppsSize(scheduler, "b1", b1Size); + assertEquals(appsPending, scheduler.getRootQueueMetrics().getAppsPending()); + } + + private void assertAppsSize(ResourceScheduler scheduler, String queueName, int size) { + assertEquals(size, scheduler.getAppsInQueue(queueName).size()); + } + + @Test + public void testMoveAppSameParent() throws Exception { + MockRM rm = setUpMove(); + AbstractYarnScheduler scheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + + // submit an app + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) + .withAppName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanagedAM(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + // check preconditions + assertOneAppInQueue(scheduler, "a1"); + assertApps(scheduler, "root", appAttemptId); + assertApps(scheduler, "a", appAttemptId); + assertApps(scheduler, "a2"); + + // now move the app + scheduler.moveApplication(app.getApplicationId(), "a2"); + + // check postconditions + assertApps(scheduler, "root", appAttemptId); + assertApps(scheduler, "a", appAttemptId); + assertApps(scheduler, "a1"); + assertOneAppInQueue(scheduler, "a2"); + + rm.stop(); + } + + private void assertApps(ResourceScheduler scheduler, + String queueName, + ApplicationAttemptId... apps) { + assertEquals(Lists.newArrayList(apps), scheduler.getAppsInQueue(queueName)); + } + + private void assertOneAppInQueue(AbstractYarnScheduler scheduler, String queueName) { + List apps = scheduler.getAppsInQueue(queueName); + assertEquals(1, apps.size()); + Assert.assertEquals(queueName, + scheduler.getApplicationAttempt(apps.get(0)).getQueue().getQueueName()); + } + + @Test + public void testMoveAppForMoveToQueueWithFreeCap() throws Exception { + + ResourceScheduler scheduler = resourceManager.getResourceScheduler(); + + NodeStatus mockNodeStatus = createMockNodeStatus(); + + // Register node1 + String host0 = "host_0"; + NodeManager nm0 = + registerNode(resourceManager, host0, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(4 * GB, 1), mockNodeStatus); + + // Register node2 + String host1 = "host_1"; + NodeManager nm1 = + registerNode(resourceManager, host1, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(2 * GB, 1), mockNodeStatus); + + // ResourceRequest priorities + Priority priority0 = Priority.newInstance(0); + Priority priority1 = Priority.newInstance(1); + + // Submit application_0 + Application application0 = + new Application("user_0", "a1", resourceManager); + application0.submit(); // app + app attempt event sent to scheduler + + application0.addNodeManager(host0, 1234, nm0); + application0.addNodeManager(host1, 1234, nm1); + + Resource capability00 = Resources.createResource(1 * GB, 1); + application0.addResourceRequestSpec(priority1, capability00); + + Resource capability01 = Resources.createResource(2 * GB, 1); + application0.addResourceRequestSpec(priority0, capability01); + + Task task00 = + new Task(application0, priority1, new String[]{host0, host1}); + application0.addTask(task00); + + // Submit application_1 + Application application1 = + new Application("user_1", "b2", resourceManager); + application1.submit(); // app + app attempt event sent to scheduler + + application1.addNodeManager(host0, 1234, nm0); + application1.addNodeManager(host1, 1234, nm1); + + Resource capability10 = Resources.createResource(1 * GB, 1); + application1.addResourceRequestSpec(priority1, capability10); + + Resource capability11 = Resources.createResource(2 * GB, 1); + application1.addResourceRequestSpec(priority0, capability11); + + Task task10 = + new Task(application1, priority1, new String[]{host0, host1}); + application1.addTask(task10); + + // Send resource requests to the scheduler + application0.schedule(); // allocate + application1.schedule(); // allocate + + // task_0_0 task_1_0 allocated, used=2G + nodeUpdate(resourceManager, nm0); + + // nothing allocated + nodeUpdate(resourceManager, nm1); + + // Get allocations from the scheduler + application0.schedule(); // task_0_0 + checkApplicationResourceUsage(1 * GB, application0); + + application1.schedule(); // task_1_0 + checkApplicationResourceUsage(1 * GB, application1); + + checkNodeResourceUsage(2 * GB, nm0); // task_0_0 (1G) and task_1_0 (1G) 2G + // available + checkNodeResourceUsage(0 * GB, nm1); // no tasks, 2G available + + // move app from a1(30% cap of total 10.5% cap) to b1(79,2% cap of 89,5% + // total cap) + scheduler.moveApplication(application0.getApplicationId(), "b1"); + + // 2GB 1C + Task task11 = + new Task(application1, priority0, + new String[]{ResourceRequest.ANY}); + application1.addTask(task11); + + application1.schedule(); + + // 2GB 1C + Task task01 = + new Task(application0, priority0, new String[]{host0, host1}); + application0.addTask(task01); + + application0.schedule(); + + // prev 2G used free 2G + nodeUpdate(resourceManager, nm0); + + // prev 0G used free 2G + nodeUpdate(resourceManager, nm1); + + // Get allocations from the scheduler + application1.schedule(); + checkApplicationResourceUsage(3 * GB, application1); + + // Get allocations from the scheduler + application0.schedule(); + checkApplicationResourceUsage(3 * GB, application0); + + checkNodeResourceUsage(4 * GB, nm0); + checkNodeResourceUsage(2 * GB, nm1); + } + + @Test + public void testMoveAppSuccess() throws Exception { + + ResourceScheduler scheduler = resourceManager.getResourceScheduler(); + + NodeStatus mockNodeStatus = createMockNodeStatus(); + + // Register node1 + String host0 = "host_0"; + NodeManager nm0 = + registerNode(resourceManager, host0, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(5 * GB, 1), mockNodeStatus); + + // Register node2 + String host1 = "host_1"; + NodeManager nm1 = + registerNode(resourceManager, host1, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(5 * GB, 1), mockNodeStatus); + + // ResourceRequest priorities + Priority priority0 = Priority.newInstance(0); + Priority priority1 = Priority.newInstance(1); + + // Submit application_0 + Application application0 = + new Application("user_0", "a1", resourceManager); + application0.submit(); // app + app attempt event sent to scheduler + + application0.addNodeManager(host0, 1234, nm0); + application0.addNodeManager(host1, 1234, nm1); + + Resource capability00 = Resources.createResource(3 * GB, 1); + application0.addResourceRequestSpec(priority1, capability00); + + Resource capability01 = Resources.createResource(2 * GB, 1); + application0.addResourceRequestSpec(priority0, capability01); + + Task task00 = + new Task(application0, priority1, new String[]{host0, host1}); + application0.addTask(task00); + + // Submit application_1 + Application application1 = + new Application("user_1", "b2", resourceManager); + application1.submit(); // app + app attempt event sent to scheduler + + application1.addNodeManager(host0, 1234, nm0); + application1.addNodeManager(host1, 1234, nm1); + + Resource capability10 = Resources.createResource(1 * GB, 1); + application1.addResourceRequestSpec(priority1, capability10); + + Resource capability11 = Resources.createResource(2 * GB, 1); + application1.addResourceRequestSpec(priority0, capability11); + + Task task10 = + new Task(application1, priority1, new String[]{host0, host1}); + application1.addTask(task10); + + // Send resource requests to the scheduler + application0.schedule(); // allocate + application1.schedule(); // allocate + + // b2 can only run 1 app at a time + scheduler.moveApplication(application0.getApplicationId(), "b2"); + + nodeUpdate(resourceManager, nm0); + + nodeUpdate(resourceManager, nm1); + + // Get allocations from the scheduler + application0.schedule(); // task_0_0 + checkApplicationResourceUsage(0 * GB, application0); + + application1.schedule(); // task_1_0 + checkApplicationResourceUsage(1 * GB, application1); + + // task_1_0 (1G) application_0 moved to b2 with max running app 1 so it is + // not scheduled + checkNodeResourceUsage(1 * GB, nm0); + checkNodeResourceUsage(0 * GB, nm1); + + // lets move application_0 to a queue where it can run + scheduler.moveApplication(application0.getApplicationId(), "a2"); + application0.schedule(); + + nodeUpdate(resourceManager, nm1); + + // Get allocations from the scheduler + application0.schedule(); // task_0_0 + checkApplicationResourceUsage(3 * GB, application0); + + checkNodeResourceUsage(1 * GB, nm0); + checkNodeResourceUsage(3 * GB, nm1); + + } + + @Test(expected = YarnException.class) + public void testMoveAppViolateQueueState() throws Exception { + resourceManager = new ResourceManager() { + @Override + protected RMNodeLabelsManager createNodeLabelManager() { + RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); + mgr.init(getConfig()); + return mgr; + } + }; + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + setupQueueConfiguration(csConf); + StringBuilder qState = new StringBuilder(); + qState.append(CapacitySchedulerConfiguration.PREFIX).append(B) + .append(CapacitySchedulerConfiguration.DOT) + .append(CapacitySchedulerConfiguration.STATE); + csConf.set(qState.toString(), QueueState.STOPPED.name()); + YarnConfiguration conf = new YarnConfiguration(csConf); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + resourceManager.init(conf); + resourceManager.getRMContext().getContainerTokenSecretManager() + .rollMasterKey(); + resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); + ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start(); + mockContext = mock(RMContext.class); + when(mockContext.getConfigurationProvider()).thenReturn( + new LocalConfigurationProvider()); + + ResourceScheduler scheduler = resourceManager.getResourceScheduler(); + + NodeStatus mockNodeStatus = createMockNodeStatus(); + + // Register node1 + String host0 = "host_0"; + NodeManager nm0 = + registerNode(resourceManager, host0, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(6 * GB, 1), mockNodeStatus); + + // ResourceRequest priorities + Priority priority0 = Priority.newInstance(0); + Priority priority1 = Priority.newInstance(1); + + // Submit application_0 + Application application0 = + new Application("user_0", "a1", resourceManager); + application0.submit(); // app + app attempt event sent to scheduler + + application0.addNodeManager(host0, 1234, nm0); + + Resource capability00 = Resources.createResource(3 * GB, 1); + application0.addResourceRequestSpec(priority1, capability00); + + Resource capability01 = Resources.createResource(2 * GB, 1); + application0.addResourceRequestSpec(priority0, capability01); + + Task task00 = + new Task(application0, priority1, new String[]{host0}); + application0.addTask(task00); + + // Send resource requests to the scheduler + application0.schedule(); // allocate + + // task_0_0 allocated + nodeUpdate(resourceManager, nm0); + + // Get allocations from the scheduler + application0.schedule(); // task_0_0 + checkApplicationResourceUsage(3 * GB, application0); + + checkNodeResourceUsage(3 * GB, nm0); + // b2 queue contains 3GB consumption app, + // add another 3GB will hit max capacity limit on queue b + scheduler.moveApplication(application0.getApplicationId(), "b1"); + + } + + @Test + public void testMoveAppQueueMetricsCheck() throws Exception { + ResourceScheduler scheduler = resourceManager.getResourceScheduler(); + + NodeStatus mockNodeStatus = createMockNodeStatus(); + + // Register node1 + String host0 = "host_0"; + NodeManager nm0 = + registerNode(resourceManager, host0, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(5 * GB, 1), mockNodeStatus); + + // Register node2 + String host1 = "host_1"; + NodeManager nm1 = + registerNode(resourceManager, host1, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(5 * GB, 1), mockNodeStatus); + + // ResourceRequest priorities + Priority priority0 = Priority.newInstance(0); + Priority priority1 = Priority.newInstance(1); + + // Submit application_0 + Application application0 = + new Application("user_0", "a1", resourceManager); + application0.submit(); // app + app attempt event sent to scheduler + + application0.addNodeManager(host0, 1234, nm0); + application0.addNodeManager(host1, 1234, nm1); + + Resource capability00 = Resources.createResource(3 * GB, 1); + application0.addResourceRequestSpec(priority1, capability00); + + Resource capability01 = Resources.createResource(2 * GB, 1); + application0.addResourceRequestSpec(priority0, capability01); + + Task task00 = + new Task(application0, priority1, new String[]{host0, host1}); + application0.addTask(task00); + + // Submit application_1 + Application application1 = + new Application("user_1", "b2", resourceManager); + application1.submit(); // app + app attempt event sent to scheduler + + application1.addNodeManager(host0, 1234, nm0); + application1.addNodeManager(host1, 1234, nm1); + + Resource capability10 = Resources.createResource(1 * GB, 1); + application1.addResourceRequestSpec(priority1, capability10); + + Resource capability11 = Resources.createResource(2 * GB, 1); + application1.addResourceRequestSpec(priority0, capability11); + + Task task10 = + new Task(application1, priority1, new String[]{host0, host1}); + application1.addTask(task10); + + // Send resource requests to the scheduler + application0.schedule(); // allocate + application1.schedule(); // allocate + + nodeUpdate(resourceManager, nm0); + + nodeUpdate(resourceManager, nm1); + + CapacityScheduler cs = + (CapacityScheduler) resourceManager.getResourceScheduler(); + CSQueue origRootQ = cs.getRootQueue(); + CapacitySchedulerInfo oldInfo = + new CapacitySchedulerInfo(origRootQ, cs); + int origNumAppsA = getNumAppsInQueue("a", origRootQ.getChildQueues()); + int origNumAppsRoot = origRootQ.getNumApplications(); + + scheduler.moveApplication(application0.getApplicationId(), "a2"); + + CSQueue newRootQ = cs.getRootQueue(); + int newNumAppsA = getNumAppsInQueue("a", newRootQ.getChildQueues()); + int newNumAppsRoot = newRootQ.getNumApplications(); + CapacitySchedulerInfo newInfo = + new CapacitySchedulerInfo(newRootQ, cs); + CapacitySchedulerLeafQueueInfo origOldA1 = + (CapacitySchedulerLeafQueueInfo) getQueueInfo("a1", oldInfo.getQueues()); + CapacitySchedulerLeafQueueInfo origNewA1 = + (CapacitySchedulerLeafQueueInfo) getQueueInfo("a1", newInfo.getQueues()); + CapacitySchedulerLeafQueueInfo targetOldA2 = + (CapacitySchedulerLeafQueueInfo) getQueueInfo("a2", oldInfo.getQueues()); + CapacitySchedulerLeafQueueInfo targetNewA2 = + (CapacitySchedulerLeafQueueInfo) getQueueInfo("a2", newInfo.getQueues()); + // originally submitted here + assertEquals(1, origOldA1.getNumApplications()); + assertEquals(1, origNumAppsA); + assertEquals(2, origNumAppsRoot); + // after the move + assertEquals(0, origNewA1.getNumApplications()); + assertEquals(1, newNumAppsA); + assertEquals(2, newNumAppsRoot); + // original consumption on a1 + assertEquals(3 * GB, origOldA1.getResourcesUsed().getMemorySize()); + assertEquals(1, origOldA1.getResourcesUsed().getvCores()); + assertEquals(0, origNewA1.getResourcesUsed().getMemorySize()); // after the move + assertEquals(0, origNewA1.getResourcesUsed().getvCores()); // after the move + // app moved here with live containers + assertEquals(3 * GB, targetNewA2.getResourcesUsed().getMemorySize()); + assertEquals(1, targetNewA2.getResourcesUsed().getvCores()); + // it was empty before the move + assertEquals(0, targetOldA2.getNumApplications()); + assertEquals(0, targetOldA2.getResourcesUsed().getMemorySize()); + assertEquals(0, targetOldA2.getResourcesUsed().getvCores()); + // after the app moved here + assertEquals(1, targetNewA2.getNumApplications()); + // 1 container on original queue before move + assertEquals(1, origOldA1.getNumContainers()); + // after the move the resource released + assertEquals(0, origNewA1.getNumContainers()); + // and moved to the new queue + assertEquals(1, targetNewA2.getNumContainers()); + // which originally didn't have any + assertEquals(0, targetOldA2.getNumContainers()); + // 1 user with 3GB + assertEquals(3 * GB, origOldA1.getUsers().getUsersList().get(0) + .getResourcesUsed().getMemorySize()); + // 1 user with 1 core + assertEquals(1, origOldA1.getUsers().getUsersList().get(0) + .getResourcesUsed().getvCores()); + // user ha no more running app in the orig queue + assertEquals(0, origNewA1.getUsers().getUsersList().size()); + // 1 user with 3GB + assertEquals(3 * GB, targetNewA2.getUsers().getUsersList().get(0) + .getResourcesUsed().getMemorySize()); + // 1 user with 1 core + assertEquals(1, targetNewA2.getUsers().getUsersList().get(0) + .getResourcesUsed().getvCores()); + + // Get allocations from the scheduler + application0.schedule(); // task_0_0 + checkApplicationResourceUsage(3 * GB, application0); + + application1.schedule(); // task_1_0 + checkApplicationResourceUsage(1 * GB, application1); + + // task_1_0 (1G) application_0 moved to b2 with max running app 1 so it is + // not scheduled + checkNodeResourceUsage(4 * GB, nm0); + checkNodeResourceUsage(0 * GB, nm1); + + } + + @Test + public void testMoveAllApps() throws Exception { + MockRM rm = setUpMove(); + AbstractYarnScheduler scheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + + // submit an app + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) + .withAppName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanagedAM(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + // check preconditions + assertOneAppInQueue(scheduler, "a1"); + assertApps(scheduler, "root", appAttemptId); + assertApps(scheduler, "a", appAttemptId); + assertApps(scheduler, "a1", appAttemptId); + assertApps(scheduler, "b1"); + assertApps(scheduler, "b"); + + // now move the app + scheduler.moveAllApps("a1", "b1"); + + // check post conditions + Thread.sleep(1000); + assertOneAppInQueue(scheduler, "b1"); + assertApps(scheduler, "root", appAttemptId); + assertApps(scheduler, "b", appAttemptId); + assertApps(scheduler, "b1", appAttemptId); + assertApps(scheduler, "a1"); + assertApps(scheduler, "a"); + + rm.stop(); + } + + @Test + public void testMoveAllAppsInvalidDestination() throws Exception { + MockRM rm = setUpMove(); + ResourceScheduler scheduler = rm.getResourceScheduler(); + + // submit an app + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) + .withAppName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanagedAM(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + // check preconditions + assertApps(scheduler, "root", appAttemptId); + assertApps(scheduler, "a", appAttemptId); + assertApps(scheduler, "a1", appAttemptId); + assertApps(scheduler, "b"); + assertApps(scheduler, "b1"); + + // now move the app + try { + scheduler.moveAllApps("a1", "DOES_NOT_EXIST"); + Assert.fail(); + } catch (YarnException e) { + // expected + } + + // check post conditions, app should still be in a1 + assertApps(scheduler, "root", appAttemptId); + assertApps(scheduler, "a", appAttemptId); + assertApps(scheduler, "a1", appAttemptId); + assertApps(scheduler, "b"); + assertApps(scheduler, "b1"); + + rm.stop(); + } + + @Test + public void testMoveAllAppsInvalidSource() throws Exception { + MockRM rm = setUpMove(); + ResourceScheduler scheduler = rm.getResourceScheduler(); + + // submit an app + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) + .withAppName("test-move-1") + .withUser("user_0") + .withAcls(null) + .withQueue("a1") + .withUnmanagedAM(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + // check preconditions + assertApps(scheduler, "root", appAttemptId); + assertApps(scheduler, "a", appAttemptId); + assertApps(scheduler, "a1", appAttemptId); + assertApps(scheduler, "b"); + assertApps(scheduler, "b1"); + + // now move the app + try { + scheduler.moveAllApps("DOES_NOT_EXIST", "b1"); + Assert.fail(); + } catch (YarnException e) { + // expected + } + + // check post conditions, app should still be in a1 + assertApps(scheduler, "root", appAttemptId); + assertApps(scheduler, "a", appAttemptId); + assertApps(scheduler, "a1", appAttemptId); + assertApps(scheduler, "b"); + assertApps(scheduler, "b1"); + + rm.stop(); + } + + @Test + public void testMoveAppWithActiveUsersWithOnlyPendingApps() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + CapacitySchedulerConfiguration newConf = + new CapacitySchedulerConfiguration(conf); + + // Define top-level queues + newConf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[]{"a", "b"}); + + newConf.setCapacity(A, 50); + newConf.setCapacity(B, 50); + + // Define 2nd-level queues + newConf.setQueues(A, new String[]{"a1"}); + newConf.setCapacity(A1, 100); + newConf.setUserLimitFactor(A1, 2.0f); + newConf.setMaximumAMResourcePercentPerPartition(A1, "", 0.1f); + + newConf.setQueues(B, new String[]{"b1"}); + newConf.setCapacity(B1, 100); + newConf.setUserLimitFactor(B1, 2.0f); + + MockRM rm = new MockRM(newConf); + rm.start(); + + CapacityScheduler scheduler = + (CapacityScheduler) rm.getResourceScheduler(); + + MockNM nm1 = rm.registerNode("h1:1234", 16 * GB); + + // submit an app + MockRMAppSubmissionData data3 = + MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) + .withAppName("test-move-1") + .withUser("u1") + .withAcls(null) + .withQueue("a1") + .withUnmanagedAM(false) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data3); + MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm1); + + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm) + .withAppName("app") + .withUser("u2") + .withAcls(null) + .withQueue("a1") + .withUnmanagedAM(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm, data2); + MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); + + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm) + .withAppName("app") + .withUser("u3") + .withAcls(null) + .withQueue("a1") + .withUnmanagedAM(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm, data1); + + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm) + .withAppName("app") + .withUser("u4") + .withAcls(null) + .withQueue("a1") + .withUnmanagedAM(false) + .build(); + RMApp app4 = MockRMAppSubmitter.submit(rm, data); + + // Each application asks 50 * 1GB containers + am1.allocate("*", 1 * GB, 50, null); + am2.allocate("*", 1 * GB, 50, null); + + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId()); + + // check preconditions + assertApps(scheduler, "root", + app3.getCurrentAppAttempt().getAppAttemptId(), + app4.getCurrentAppAttempt().getAppAttemptId(), + appAttemptId, + app2.getCurrentAppAttempt().getAppAttemptId()); + assertApps(scheduler, "a", + app3.getCurrentAppAttempt().getAppAttemptId(), + app4.getCurrentAppAttempt().getAppAttemptId(), + appAttemptId, + app2.getCurrentAppAttempt().getAppAttemptId()); + assertApps(scheduler, "a1", + app3.getCurrentAppAttempt().getAppAttemptId(), + app4.getCurrentAppAttempt().getAppAttemptId(), + appAttemptId, + app2.getCurrentAppAttempt().getAppAttemptId()); + assertApps(scheduler, "b"); + assertApps(scheduler, "b1"); + + UsersManager um = + (UsersManager) scheduler.getQueue("a1").getAbstractUsersManager(); + + assertEquals(4, um.getNumActiveUsers()); + assertEquals(2, um.getNumActiveUsersWithOnlyPendingApps()); + + // now move the app + scheduler.moveAllApps("a1", "b1"); + + //Triggering this event so that user limit computation can + //happen again + for (int i = 0; i < 10; i++) { + cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); + Thread.sleep(500); + } + + // check post conditions + assertApps(scheduler, "root", + appAttemptId, + app2.getCurrentAppAttempt().getAppAttemptId(), + app3.getCurrentAppAttempt().getAppAttemptId(), + app4.getCurrentAppAttempt().getAppAttemptId()); + assertApps(scheduler, "a"); + assertApps(scheduler, "a1"); + assertApps(scheduler, "b", + appAttemptId, + app2.getCurrentAppAttempt().getAppAttemptId(), + app3.getCurrentAppAttempt().getAppAttemptId(), + app4.getCurrentAppAttempt().getAppAttemptId()); + assertApps(scheduler, "b1", + appAttemptId, + app2.getCurrentAppAttempt().getAppAttemptId(), + app3.getCurrentAppAttempt().getAppAttemptId(), + app4.getCurrentAppAttempt().getAppAttemptId()); + + UsersManager umB1 = + (UsersManager) scheduler.getQueue("b1").getAbstractUsersManager(); + + assertEquals(2, umB1.getNumActiveUsers()); + assertEquals(2, umB1.getNumActiveUsersWithOnlyPendingApps()); + + rm.close(); + } + + @Test(timeout = 60000) + public void testMoveAttemptNotAdded() throws Exception { + Configuration conf = new Configuration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + MockRM rm = new MockRM(getCapacityConfiguration(conf)); + rm.start(); + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + ApplicationId appId = BuilderUtils.newApplicationId(100, 1); + ApplicationAttemptId appAttemptId = + BuilderUtils.newApplicationAttemptId(appId, 1); + + RMAppAttemptMetrics attemptMetric = + new RMAppAttemptMetrics(appAttemptId, rm.getRMContext()); + RMAppImpl app = mock(RMAppImpl.class); + when(app.getApplicationId()).thenReturn(appId); + RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class); + Container container = mock(Container.class); + when(attempt.getMasterContainer()).thenReturn(container); + ApplicationSubmissionContext submissionContext = + mock(ApplicationSubmissionContext.class); + when(attempt.getSubmissionContext()).thenReturn(submissionContext); + when(attempt.getAppAttemptId()).thenReturn(appAttemptId); + when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric); + when(app.getCurrentAppAttempt()).thenReturn(attempt); + + rm.getRMContext().getRMApps().put(appId, app); + + SchedulerEvent addAppEvent = + new AppAddedSchedulerEvent(appId, "a1", "user"); + try { + cs.moveApplication(appId, "b1"); + fail("Move should throw exception app not available"); + } catch (YarnException e) { + assertEquals("App to be moved application_100_0001 not found.", + e.getMessage()); + } + cs.handle(addAppEvent); + cs.moveApplication(appId, "b1"); + SchedulerEvent addAttemptEvent = + new AppAttemptAddedSchedulerEvent(appAttemptId, false); + cs.handle(addAttemptEvent); + CSQueue rootQ = cs.getRootQueue(); + CSQueue queueB = cs.getQueue("b"); + CSQueue queueA = cs.getQueue("a"); + CSQueue queueA1 = cs.getQueue("a1"); + CSQueue queueB1 = cs.getQueue("b1"); + Assert.assertEquals(1, rootQ.getNumApplications()); + Assert.assertEquals(0, queueA.getNumApplications()); + Assert.assertEquals(1, queueB.getNumApplications()); + Assert.assertEquals(0, queueA1.getNumApplications()); + Assert.assertEquals(1, queueB1.getNumApplications()); + + rm.close(); + } + + @Test + public void testRemoveAttemptMoveAdded() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + CapacityScheduler.class); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); + // Create Mock RM + MockRM rm = new MockRM(getCapacityConfiguration(conf)); + CapacityScheduler sch = (CapacityScheduler) rm.getResourceScheduler(); + // add node + Resource newResource = Resource.newInstance(4 * GB, 1); + RMNode node = MockNodes.newNodeInfo(0, newResource, 1, "127.0.0.1"); + SchedulerEvent addNode = new NodeAddedSchedulerEvent(node); + sch.handle(addNode); + + ApplicationAttemptId appAttemptId = appHelper(rm, sch, 100, 1, "a1", "user"); + + // get Queues + CSQueue queueA1 = sch.getQueue("a1"); + CSQueue queueB = sch.getQueue("b"); + CSQueue queueB1 = sch.getQueue("b1"); + + // add Running rm container and simulate live containers to a1 + ContainerId newContainerId = ContainerId.newContainerId(appAttemptId, 2); + RMContainerImpl rmContainer = mock(RMContainerImpl.class); + when(rmContainer.getState()).thenReturn(RMContainerState.RUNNING); + Container container2 = mock(Container.class); + when(rmContainer.getContainer()).thenReturn(container2); + Resource resource = Resource.newInstance(1024, 1); + when(container2.getResource()).thenReturn(resource); + when(rmContainer.getExecutionType()).thenReturn(ExecutionType.GUARANTEED); + when(container2.getNodeId()).thenReturn(node.getNodeID()); + when(container2.getId()).thenReturn(newContainerId); + when(rmContainer.getNodeLabelExpression()) + .thenReturn(RMNodeLabelsManager.NO_LABEL); + when(rmContainer.getContainerId()).thenReturn(newContainerId); + sch.getApplicationAttempt(appAttemptId).getLiveContainersMap() + .put(newContainerId, rmContainer); + QueueMetrics queueA1M = queueA1.getMetrics(); + queueA1M.incrPendingResources(rmContainer.getNodeLabelExpression(), + "user1", 1, resource); + queueA1M.allocateResources(rmContainer.getNodeLabelExpression(), + "user1", resource); + // remove attempt + sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId, + RMAppAttemptState.KILLED, true)); + // Move application to queue b1 + sch.moveApplication(appAttemptId.getApplicationId(), "b1"); + // Check queue metrics after move + Assert.assertEquals(0, queueA1.getNumApplications()); + Assert.assertEquals(1, queueB.getNumApplications()); + Assert.assertEquals(0, queueB1.getNumApplications()); + + // Release attempt add event + ApplicationAttemptId appAttemptId2 = + BuilderUtils.newApplicationAttemptId(appAttemptId.getApplicationId(), 2); + SchedulerEvent addAttemptEvent2 = + new AppAttemptAddedSchedulerEvent(appAttemptId2, true); + sch.handle(addAttemptEvent2); + + // Check metrics after attempt added + Assert.assertEquals(0, queueA1.getNumApplications()); + Assert.assertEquals(1, queueB.getNumApplications()); + Assert.assertEquals(1, queueB1.getNumApplications()); + + + QueueMetrics queueB1M = queueB1.getMetrics(); + QueueMetrics queueBM = queueB.getMetrics(); + // Verify allocation MB of current state + Assert.assertEquals(0, queueA1M.getAllocatedMB()); + Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores()); + Assert.assertEquals(1024, queueB1M.getAllocatedMB()); + Assert.assertEquals(1, queueB1M.getAllocatedVirtualCores()); + + // remove attempt + sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId2, + RMAppAttemptState.FINISHED, false)); + + Assert.assertEquals(0, queueA1M.getAllocatedMB()); + Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores()); + Assert.assertEquals(0, queueB1M.getAllocatedMB()); + Assert.assertEquals(0, queueB1M.getAllocatedVirtualCores()); + + verifyQueueMetrics(queueB1M); + verifyQueueMetrics(queueBM); + // Verify queue A1 metrics + verifyQueueMetrics(queueA1M); + rm.close(); + } + + @Test + public void testAppSubmission() throws Exception { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.setQueues(A, new String[]{"a1", "a2", "b"}); + conf.setCapacity(A1, 20); + conf.setCapacity("root.a.b", 10); + MockRM rm = new MockRM(conf); + rm.start(); + + RMApp noParentQueueApp = submitAppAndWaitForState(rm, "q", RMAppState.FAILED); + Assert.assertEquals(RMAppState.FAILED, noParentQueueApp.getState()); + + RMApp ambiguousQueueApp = submitAppAndWaitForState(rm, "b", RMAppState.FAILED); + Assert.assertEquals(RMAppState.FAILED, ambiguousQueueApp.getState()); + + RMApp emptyPartQueueApp = submitAppAndWaitForState(rm, "root..a1", RMAppState.FAILED); + Assert.assertEquals(RMAppState.FAILED, emptyPartQueueApp.getState()); + + RMApp failedAutoQueue = submitAppAndWaitForState(rm, "root.a.b.c.d", RMAppState.FAILED); + Assert.assertEquals(RMAppState.FAILED, failedAutoQueue.getState()); + } + + private RMApp submitAppAndWaitForState(MockRM rm, String b, RMAppState state) throws Exception { + MockRMAppSubmissionData ambiguousQueueAppData = + MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) + .withWaitForAppAcceptedState(false) + .withAppName("app") + .withUser("user") + .withAcls(null) + .withQueue(b) + .withUnmanagedAM(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, ambiguousQueueAppData); + rm.waitForState(app1.getApplicationId(), state); + return app1; + } + + private int getNumAppsInQueue(String name, List queues) { + for (CSQueue queue : queues) { + if (queue.getQueueShortName().equals(name)) { + return queue.getNumApplications(); + } + } + return -1; + } + + private CapacitySchedulerQueueInfo getQueueInfo(String name, + CapacitySchedulerQueueInfoList info) { + if (info != null) { + for (CapacitySchedulerQueueInfo queueInfo : info.getQueueInfoList()) { + if (queueInfo.getQueueName().equals(name)) { + return queueInfo; + } else { + CapacitySchedulerQueueInfo result = + getQueueInfo(name, queueInfo.getQueues()); + if (result == null) { + continue; + } + return result; + } + } + } + return null; + } + + private void verifyQueueMetrics(QueueMetrics queue) { + Assert.assertEquals(0, queue.getPendingMB()); + Assert.assertEquals(0, queue.getActiveUsers()); + Assert.assertEquals(0, queue.getActiveApps()); + Assert.assertEquals(0, queue.getAppsPending()); + Assert.assertEquals(0, queue.getAppsRunning()); + Assert.assertEquals(0, queue.getAllocatedMB()); + Assert.assertEquals(0, queue.getAllocatedVirtualCores()); + } + + private Configuration getCapacityConfiguration(Configuration config) { + CapacitySchedulerConfiguration conf = + new CapacitySchedulerConfiguration(config); + + // Define top-level queues + conf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[]{"a", "b"}); + conf.setCapacity(A, 50); + conf.setCapacity(B, 50); + conf.setQueues(A, new String[]{"a1", "a2"}); + conf.setCapacity(A1, 50); + conf.setCapacity(A2, 50); + conf.setQueues(B, new String[]{"b1"}); + conf.setCapacity(B1, 100); + return conf; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java new file mode 100644 index 00000000000000..c557354b6f414f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java @@ -0,0 +1,387 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import java.util.Collections; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.event.Event; +import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.api.records.NodeStatus; +import org.apache.hadoop.yarn.server.resourcemanager.Application; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; +import org.apache.hadoop.yarn.server.resourcemanager.NodeManager; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.Task; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimpleCandidateNodeSet; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import static org.apache.hadoop.yarn.server.resourcemanager.MockNM.createMockNodeStatus; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.appHelper; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createResourceManager; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.nodeUpdate; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.registerNode; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.stopResourceManager; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerAutoCreatedQueueBase.NULL_UPDATE_REQUESTS; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TestCapacitySchedulerNodes { + + private ResourceManager resourceManager = null; + + @Before + public void setUp() throws Exception { + resourceManager = createResourceManager(); + } + + @After + public void tearDown() throws Exception { + stopResourceManager(resourceManager); + } + + @Test + public void testReconnectedNode() throws Exception { + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + setupQueueConfiguration(csConf); + CapacityScheduler cs = new CapacityScheduler(); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + cs.init(csConf); + cs.start(); + cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, + null, null, new RMContainerTokenSecretManager(csConf), + new NMTokenSecretManagerInRM(csConf), + new ClientToAMTokenSecretManagerInRM(), null)); + + RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1); + RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2); + + cs.handle(new NodeAddedSchedulerEvent(n1)); + cs.handle(new NodeAddedSchedulerEvent(n2)); + + Assert.assertEquals(6 * GB, cs.getClusterResource().getMemorySize()); + + // reconnect n1 with downgraded memory + n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1); + cs.handle(new NodeRemovedSchedulerEvent(n1)); + cs.handle(new NodeAddedSchedulerEvent(n1)); + + Assert.assertEquals(4 * GB, cs.getClusterResource().getMemorySize()); + cs.stop(); + } + + @Test + public void testBlackListNodes() throws Exception { + Configuration conf = new Configuration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + MockRM rm = new MockRM(conf); + rm.start(); + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + String host = "127.0.0.1"; + RMNode node = + MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host); + cs.handle(new NodeAddedSchedulerEvent(node)); + + ApplicationAttemptId appAttemptId = appHelper(rm, cs, 100, 1, "default", "user"); + + // Verify the blacklist can be updated independent of requesting containers + cs.allocate(appAttemptId, Collections.emptyList(), null, + Collections.emptyList(), + Collections.singletonList(host), null, NULL_UPDATE_REQUESTS); + Assert.assertTrue(cs.getApplicationAttempt(appAttemptId) + .isPlaceBlacklisted(host)); + cs.allocate(appAttemptId, Collections.emptyList(), null, + Collections.emptyList(), null, + Collections.singletonList(host), NULL_UPDATE_REQUESTS); + Assert.assertFalse(cs.getApplicationAttempt(appAttemptId) + .isPlaceBlacklisted(host)); + rm.stop(); + } + + @Test + public void testNumClusterNodes() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + CapacityScheduler cs = new CapacityScheduler(); + cs.setConf(conf); + RMContext rmContext = TestUtils.getMockRMContext(); + cs.setRMContext(rmContext); + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + setupQueueConfiguration(csConf); + cs.init(csConf); + cs.start(); + assertEquals(0, cs.getNumClusterNodes()); + + RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1); + RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2); + cs.handle(new NodeAddedSchedulerEvent(n1)); + cs.handle(new NodeAddedSchedulerEvent(n2)); + assertEquals(2, cs.getNumClusterNodes()); + + cs.handle(new NodeRemovedSchedulerEvent(n1)); + assertEquals(1, cs.getNumClusterNodes()); + cs.handle(new NodeAddedSchedulerEvent(n1)); + assertEquals(2, cs.getNumClusterNodes()); + cs.handle(new NodeRemovedSchedulerEvent(n2)); + cs.handle(new NodeRemovedSchedulerEvent(n1)); + assertEquals(0, cs.getNumClusterNodes()); + + cs.stop(); + } + + @Test + public void testDefaultNodeLabelExpressionQueueConfig() throws Exception { + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + conf.setDefaultNodeLabelExpression("root.a", " x"); + conf.setDefaultNodeLabelExpression("root.b", " y "); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + cs.init(conf); + cs.start(); + + QueueInfo queueInfoA = cs.getQueueInfo("a", true, false); + Assert.assertEquals("Queue Name should be a", "a", + queueInfoA.getQueueName()); + Assert.assertEquals("Queue Path should be root.a", "root.a", + queueInfoA.getQueuePath()); + Assert.assertEquals("Default Node Label Expression should be x", "x", + queueInfoA.getDefaultNodeLabelExpression()); + + QueueInfo queueInfoB = cs.getQueueInfo("b", true, false); + Assert.assertEquals("Queue Name should be b", "b", + queueInfoB.getQueueName()); + Assert.assertEquals("Queue Path should be root.b", "root.b", + queueInfoB.getQueuePath()); + Assert.assertEquals("Default Node Label Expression should be y", "y", + queueInfoB.getDefaultNodeLabelExpression()); + } + + @Test + public void testRemovedNodeDecommissioningNode() throws Exception { + NodeStatus mockNodeStatus = createMockNodeStatus(); + + // Register nodemanager + NodeManager nm = registerNode(resourceManager, "host_decom", 1234, 2345, + NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4), + mockNodeStatus); + + RMNode node = + resourceManager.getRMContext().getRMNodes().get(nm.getNodeId()); + // Send a heartbeat to kick the tires on the Scheduler + NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); + resourceManager.getResourceScheduler().handle(nodeUpdate); + + // force remove the node to simulate race condition + ((CapacityScheduler) resourceManager.getResourceScheduler()).getNodeTracker(). + removeNode(nm.getNodeId()); + // Kick off another heartbeat with the node state mocked to decommissioning + RMNode spyNode = + Mockito.spy(resourceManager.getRMContext().getRMNodes() + .get(nm.getNodeId())); + when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING); + resourceManager.getResourceScheduler().handle( + new NodeUpdateSchedulerEvent(spyNode)); + } + + @Test + public void testResourceUpdateDecommissioningNode() throws Exception { + // Mock the RMNodeResourceUpdate event handler to update SchedulerNode + // to have 0 available resource + RMContext spyContext = Mockito.spy(resourceManager.getRMContext()); + Dispatcher mockDispatcher = mock(AsyncDispatcher.class); + when(mockDispatcher.getEventHandler()).thenReturn(new EventHandler() { + @Override + public void handle(Event event) { + if (event instanceof RMNodeResourceUpdateEvent) { + RMNodeResourceUpdateEvent resourceEvent = + (RMNodeResourceUpdateEvent) event; + resourceManager + .getResourceScheduler() + .getSchedulerNode(resourceEvent.getNodeId()) + .updateTotalResource(resourceEvent.getResourceOption().getResource()); + } + } + }); + Mockito.doReturn(mockDispatcher).when(spyContext).getDispatcher(); + ((CapacityScheduler) resourceManager.getResourceScheduler()) + .setRMContext(spyContext); + ((AsyncDispatcher) mockDispatcher).start(); + + NodeStatus mockNodeStatus = createMockNodeStatus(); + + // Register node + String host0 = "host_0"; + NodeManager nm0 = registerNode(resourceManager, host0, 1234, 2345, + NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4), + mockNodeStatus); + // ResourceRequest priorities + Priority priority0 = Priority.newInstance(0); + + // Submit an application + Application application0 = + new Application("user_0", "a1", resourceManager); + application0.submit(); + + application0.addNodeManager(host0, 1234, nm0); + + Resource capability00 = Resources.createResource(1 * GB, 1); + application0.addResourceRequestSpec(priority0, capability00); + + Task task00 = + new Task(application0, priority0, new String[]{host0}); + application0.addTask(task00); + + // Send resource requests to the scheduler + application0.schedule(); + + nodeUpdate(resourceManager, nm0); + // Kick off another heartbeat with the node state mocked to decommissioning + // This should update the schedulernodes to have 0 available resource + RMNode spyNode = + Mockito.spy(resourceManager.getRMContext().getRMNodes() + .get(nm0.getNodeId())); + when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING); + resourceManager.getResourceScheduler().handle( + new NodeUpdateSchedulerEvent(spyNode)); + + // Get allocations from the scheduler + application0.schedule(); + + // Check the used resource is 1 GB 1 core + Assert.assertEquals(1 * GB, nm0.getUsed().getMemorySize()); + Resource usedResource = + resourceManager.getResourceScheduler() + .getSchedulerNode(nm0.getNodeId()).getAllocatedResource(); + Assert.assertEquals("Used Resource Memory Size should be 1GB", 1 * GB, + usedResource.getMemorySize()); + Assert.assertEquals("Used Resource Virtual Cores should be 1", 1, + usedResource.getVirtualCores()); + // Check total resource of scheduler node is also changed to 1 GB 1 core + Resource totalResource = + resourceManager.getResourceScheduler() + .getSchedulerNode(nm0.getNodeId()).getTotalResource(); + Assert.assertEquals("Total Resource Memory Size should be 1GB", 1 * GB, + totalResource.getMemorySize()); + Assert.assertEquals("Total Resource Virtual Cores should be 1", 1, + totalResource.getVirtualCores()); + // Check the available resource is 0/0 + Resource availableResource = + resourceManager.getResourceScheduler() + .getSchedulerNode(nm0.getNodeId()).getUnallocatedResource(); + Assert.assertEquals("Available Resource Memory Size should be 0", 0, + availableResource.getMemorySize()); + Assert.assertEquals("Available Resource Memory Size should be 0", 0, + availableResource.getVirtualCores()); + // Kick off another heartbeat where the RMNodeResourceUpdateEvent would + // be skipped for DECOMMISSIONING state since the total resource is + // already equal to used resource from the previous heartbeat. + when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING); + resourceManager.getResourceScheduler().handle( + new NodeUpdateSchedulerEvent(spyNode)); + verify(mockDispatcher, times(4)).getEventHandler(); + } + + @Test + public void testSchedulingOnRemovedNode() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.setBoolean( + CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, + false); + + MockRM rm = new MockRM(conf); + rm.start(); + RMApp app = MockRMAppSubmitter.submitWithMemory(100, rm); + rm.drainEvents(); + + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10240, 10); + MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); + + //remove nm2 to keep am alive + MockNM nm2 = rm.registerNode("127.0.0.1:1235", 10240, 10); + + am.allocate(ResourceRequest.ANY, 2048, 1, null); + + CapacityScheduler scheduler = + (CapacityScheduler) rm.getRMContext().getScheduler(); + FiCaSchedulerNode node = + (FiCaSchedulerNode) + scheduler.getNodeTracker().getNode(nm2.getNodeId()); + scheduler.handle(new NodeRemovedSchedulerEvent( + rm.getRMContext().getRMNodes().get(nm2.getNodeId()))); + // schedulerNode is removed, try allocate a container + scheduler.allocateContainersToNode(new SimpleCandidateNodeSet<>(node), + true); + + AppAttemptRemovedSchedulerEvent appRemovedEvent1 = + new AppAttemptRemovedSchedulerEvent( + am.getApplicationAttemptId(), + RMAppAttemptState.FINISHED, false); + scheduler.handle(appRemovedEvent1); + rm.stop(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java new file mode 100644 index 00000000000000..fc1870097e9a4f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java @@ -0,0 +1,888 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import java.io.IOException; +import java.util.Map; + +import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocMb; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocVcores; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocation; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.unsetMaxAllocation; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.checkQueueStructureCapacities; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ExpectedCapacities; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.findQueue; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.getDefaultCapacities; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfWithoutChildrenOfB; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithB1AsParentQueue; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithoutB; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithoutB1; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createMockRMContext; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createResourceManager; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.stopResourceManager; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.when; + +public class TestCapacitySchedulerQueues { + + private static final Logger LOG = + LoggerFactory.getLogger(TestCapacitySchedulerQueues.class); + private ResourceManager resourceManager = null; + private RMContext mockContext; + + @Before + public void setUp() throws Exception { + resourceManager = createResourceManager(); + mockContext = createMockRMContext(); + } + + @After + public void tearDown() throws Exception { + stopResourceManager(resourceManager); + } + + /** + * Test that parseQueue throws an exception when two leaf queues have the + * same name. + * + * @throws IOException + */ + @Test(expected = IOException.class) + public void testParseQueue() throws IOException { + CapacityScheduler cs = new CapacityScheduler(); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + cs.init(conf); + cs.start(); + + conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[]{"b1"}); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); + conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); + + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new NMTokenSecretManagerInRM(conf), + new ClientToAMTokenSecretManagerInRM(), null)); + } + + @Test + public void testRefreshQueues() throws Exception { + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new NMTokenSecretManagerInRM(conf), + new ClientToAMTokenSecretManagerInRM(), null); + setupQueueConfiguration(conf); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + cs.init(conf); + cs.start(); + cs.reinitialize(conf, rmContext); + checkQueueStructureCapacities(cs); + + conf.setCapacity(A, 80f); + conf.setCapacity(B, 20f); + cs.reinitialize(conf, mockContext); + checkQueueStructureCapacities(cs, getDefaultCapacities(80f / 100.0f, 20f / 100.0f)); + cs.stop(); + } + + @Test + public void testRefreshQueuesWithNewQueue() throws Exception { + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + cs.init(conf); + cs.start(); + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new NMTokenSecretManagerInRM(conf), + new ClientToAMTokenSecretManagerInRM(), null)); + checkQueueStructureCapacities(cs); + + // Add a new queue b4 + final String b4 = B + ".b4"; + final float b4Capacity = 10; + final float modifiedB3Capacity = B3_CAPACITY - b4Capacity; + + try { + conf.setCapacity(A, 80f); + conf.setCapacity(B, 20f); + conf.setQueues(B, new String[]{"b1", "b2", "b3", "b4"}); + conf.setCapacity(B1, B1_CAPACITY); + conf.setCapacity(B2, B2_CAPACITY); + conf.setCapacity(B3, modifiedB3Capacity); + conf.setCapacity(b4, b4Capacity); + cs.reinitialize(conf, mockContext); + + final float capA = 80f / 100.0f; + final float capB = 20f / 100.0f; + Map expectedCapacities = + getDefaultCapacities(capA, capB); + expectedCapacities.put(B3, + new ExpectedCapacities(modifiedB3Capacity / 100.0f, capB)); + expectedCapacities.put(b4, new ExpectedCapacities(b4Capacity / 100.0f, capB)); + checkQueueStructureCapacities(cs, expectedCapacities); + + // Verify parent for B4 + CSQueue rootQueue = cs.getRootQueue(); + CSQueue queueB = findQueue(rootQueue, B); + CSQueue queueB4 = findQueue(queueB, b4); + + assertEquals(queueB, queueB4.getParent()); + } finally { + cs.stop(); + } + } + + @Test + public void testRefreshQueuesMaxAllocationRefresh() throws Exception { + // queue refresh should not allow changing the maximum allocation setting + // per queue to be smaller than previous setting + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + cs.init(conf); + cs.start(); + cs.reinitialize(conf, mockContext); + checkQueueStructureCapacities(cs); + + assertEquals("max allocation in CS", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumResourceCapability().getMemorySize()); + assertEquals("max allocation for A1", + Resources.none(), + conf.getQueueMaximumAllocation(A1)); + assertEquals("max allocation", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize()); + + CSQueue rootQueue = cs.getRootQueue(); + CSQueue queueA = findQueue(rootQueue, A); + CSQueue queueA1 = findQueue(queueA, A1); + assertEquals("queue max allocation", ((LeafQueue) queueA1) + .getMaximumAllocation().getMemorySize(), 8192); + + setMaxAllocMb(conf, A1, 4096); + + try { + cs.reinitialize(conf, mockContext); + fail("should have thrown exception"); + } catch (IOException e) { + assertTrue("max allocation exception", + e.getCause().toString().contains("not be decreased")); + } + + setMaxAllocMb(conf, A1, 8192); + cs.reinitialize(conf, mockContext); + + setMaxAllocVcores(conf, A1, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES - 1); + try { + cs.reinitialize(conf, mockContext); + fail("should have thrown exception"); + } catch (IOException e) { + assertTrue("max allocation exception", + e.getCause().toString().contains("not be decreased")); + } + } + + @Test + public void testRefreshQueuesMaxAllocationPerQueueLarge() throws Exception { + // verify we can't set the allocation per queue larger then cluster setting + CapacityScheduler cs = new CapacityScheduler(); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + cs.init(conf); + cs.start(); + // change max allocation for B3 queue to be larger then cluster max + setMaxAllocMb(conf, B3, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 2048); + try { + cs.reinitialize(conf, mockContext); + fail("should have thrown exception"); + } catch (IOException e) { + assertTrue("maximum allocation exception", + e.getCause().getMessage().contains("maximum allocation")); + } + + setMaxAllocMb(conf, B3, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); + cs.reinitialize(conf, mockContext); + + setMaxAllocVcores(conf, B3, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1); + try { + cs.reinitialize(conf, mockContext); + fail("should have thrown exception"); + } catch (IOException e) { + assertTrue("maximum allocation exception", + e.getCause().getMessage().contains("maximum allocation")); + } + } + + @Test + public void testRefreshQueuesMaxAllocationRefreshLarger() throws Exception { + // queue refresh should allow max allocation per queue to go larger + CapacityScheduler cs = new CapacityScheduler(); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + setMaxAllocMb(conf, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); + setMaxAllocVcores(conf, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + setMaxAllocMb(conf, A1, 4096); + setMaxAllocVcores(conf, A1, 2); + cs.init(conf); + cs.start(); + cs.reinitialize(conf, mockContext); + checkQueueStructureCapacities(cs); + + CSQueue rootQueue = cs.getRootQueue(); + CSQueue queueA = findQueue(rootQueue, A); + CSQueue queueA1 = findQueue(queueA, A1); + + assertEquals("max capability MB in CS", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumResourceCapability().getMemorySize()); + assertEquals("max capability vcores in CS", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + cs.getMaximumResourceCapability().getVirtualCores()); + assertEquals("max allocation MB A1", + 4096, + queueA1.getMaximumAllocation().getMemorySize()); + assertEquals("max allocation vcores A1", + 2, + queueA1.getMaximumAllocation().getVirtualCores()); + assertEquals("cluster max allocation MB", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize()); + assertEquals("cluster max allocation vcores", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getVirtualCores()); + + assertEquals("queue max allocation", 4096, + queueA1.getMaximumAllocation().getMemorySize()); + + setMaxAllocMb(conf, A1, 6144); + setMaxAllocVcores(conf, A1, 3); + cs.reinitialize(conf, null); + // conf will have changed but we shouldn't be able to change max allocation + // for the actual queue + assertEquals("max allocation MB A1", 6144, + queueA1.getMaximumAllocation().getMemorySize()); + assertEquals("max allocation vcores A1", 3, + queueA1.getMaximumAllocation().getVirtualCores()); + assertEquals("max allocation MB cluster", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize()); + assertEquals("max allocation vcores cluster", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getVirtualCores()); + assertEquals("queue max allocation MB", 6144, + queueA1.getMaximumAllocation().getMemorySize()); + assertEquals("queue max allocation vcores", 3, + queueA1.getMaximumAllocation().getVirtualCores()); + assertEquals("max capability MB cluster", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumResourceCapability().getMemorySize()); + assertEquals("cluster max capability vcores", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + cs.getMaximumResourceCapability().getVirtualCores()); + } + + @Test + public void testRefreshQueuesMaxAllocationCSError() throws Exception { + // Try to refresh the cluster level max allocation size to be smaller + // and it should error out + CapacityScheduler cs = new CapacityScheduler(); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + setMaxAllocMb(conf, 10240); + setMaxAllocVcores(conf, 10); + setMaxAllocMb(conf, A1, 4096); + setMaxAllocVcores(conf, A1, 4); + cs.init(conf); + cs.start(); + cs.reinitialize(conf, mockContext); + checkQueueStructureCapacities(cs); + + assertEquals("max allocation MB in CS", 10240, + cs.getMaximumResourceCapability().getMemorySize()); + assertEquals("max allocation vcores in CS", 10, + cs.getMaximumResourceCapability().getVirtualCores()); + + setMaxAllocMb(conf, 6144); + try { + cs.reinitialize(conf, mockContext); + fail("should have thrown exception"); + } catch (IOException e) { + assertTrue("max allocation exception", + e.getCause().toString().contains("not be decreased")); + } + + setMaxAllocMb(conf, 10240); + cs.reinitialize(conf, mockContext); + + setMaxAllocVcores(conf, 8); + try { + cs.reinitialize(conf, mockContext); + fail("should have thrown exception"); + } catch (IOException e) { + assertTrue("max allocation exception", + e.getCause().toString().contains("not be decreased")); + } + } + + @Test + public void testRefreshQueuesMaxAllocationCSLarger() throws Exception { + // Try to refresh the cluster level max allocation size to be larger + // and verify that if there is no setting per queue it uses the + // cluster level setting. + CapacityScheduler cs = new CapacityScheduler(); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + setMaxAllocMb(conf, 10240); + setMaxAllocVcores(conf, 10); + setMaxAllocMb(conf, A1, 4096); + setMaxAllocVcores(conf, A1, 4); + cs.init(conf); + cs.start(); + cs.reinitialize(conf, mockContext); + checkQueueStructureCapacities(cs); + + assertEquals("max allocation MB in CS", 10240, + cs.getMaximumResourceCapability().getMemorySize()); + assertEquals("max allocation vcores in CS", 10, + cs.getMaximumResourceCapability().getVirtualCores()); + + CSQueue rootQueue = cs.getRootQueue(); + CSQueue queueA = findQueue(rootQueue, A); + CSQueue queueB = findQueue(rootQueue, B); + CSQueue queueA1 = findQueue(queueA, A1); + CSQueue queueA2 = findQueue(queueA, A2); + CSQueue queueB2 = findQueue(queueB, B2); + + assertEquals("queue A1 max allocation MB", 4096, + queueA1.getMaximumAllocation().getMemorySize()); + assertEquals("queue A1 max allocation vcores", 4, + queueA1.getMaximumAllocation().getVirtualCores()); + assertEquals("queue A2 max allocation MB", 10240, + queueA2.getMaximumAllocation().getMemorySize()); + assertEquals("queue A2 max allocation vcores", 10, + queueA2.getMaximumAllocation().getVirtualCores()); + assertEquals("queue B2 max allocation MB", 10240, + queueB2.getMaximumAllocation().getMemorySize()); + assertEquals("queue B2 max allocation vcores", 10, + queueB2.getMaximumAllocation().getVirtualCores()); + + setMaxAllocMb(conf, 12288); + setMaxAllocVcores(conf, 12); + cs.reinitialize(conf, null); + // cluster level setting should change and any queues without + // per queue setting + assertEquals("max allocation MB in CS", 12288, + cs.getMaximumResourceCapability().getMemorySize()); + assertEquals("max allocation vcores in CS", 12, + cs.getMaximumResourceCapability().getVirtualCores()); + assertEquals("queue A1 max MB allocation", 4096, + queueA1.getMaximumAllocation().getMemorySize()); + assertEquals("queue A1 max vcores allocation", 4, + queueA1.getMaximumAllocation().getVirtualCores()); + assertEquals("queue A2 max MB allocation", 12288, + queueA2.getMaximumAllocation().getMemorySize()); + assertEquals("queue A2 max vcores allocation", 12, + queueA2.getMaximumAllocation().getVirtualCores()); + assertEquals("queue B2 max MB allocation", 12288, + queueB2.getMaximumAllocation().getMemorySize()); + assertEquals("queue B2 max vcores allocation", 12, + queueB2.getMaximumAllocation().getVirtualCores()); + } + + /** + * Test for queue deletion. + * + * @throws Exception + */ + @Test + public void testRefreshQueuesWithQueueDelete() throws Exception { + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new NMTokenSecretManagerInRM(conf), + new ClientToAMTokenSecretManagerInRM(), null); + setupQueueConfiguration(conf); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + cs.init(conf); + cs.start(); + cs.reinitialize(conf, rmContext); + checkQueueStructureCapacities(cs); + + // test delete leaf queue when there is application running. + Map queues = + cs.getCapacitySchedulerQueueManager().getShortNameQueues(); + String b1QTobeDeleted = "b1"; + LeafQueue csB1Queue = Mockito.spy((LeafQueue) queues.get(b1QTobeDeleted)); + when(csB1Queue.getState()).thenReturn(QueueState.DRAINING) + .thenReturn(QueueState.STOPPED); + cs.getCapacitySchedulerQueueManager().addQueue(b1QTobeDeleted, csB1Queue); + conf = new CapacitySchedulerConfiguration(); + setupQueueConfigurationWithoutB1(conf); + try { + cs.reinitialize(conf, mockContext); + fail("Expected to throw exception when refresh queue tries to delete a" + + " queue with running apps"); + } catch (IOException e) { + // ignore + } + + // test delete leaf queue(root.b.b1) when there is no application running. + conf = new CapacitySchedulerConfiguration(); + setupQueueConfigurationWithoutB1(conf); + try { + cs.reinitialize(conf, mockContext); + } catch (IOException e) { + LOG.error( + "Expected to NOT throw exception when refresh queue tries to delete" + + " a queue WITHOUT running apps", + e); + fail("Expected to NOT throw exception when refresh queue tries to delete" + + " a queue WITHOUT running apps"); + } + CSQueue rootQueue = cs.getRootQueue(); + CSQueue queueB = findQueue(rootQueue, B); + CSQueue queueB3 = findQueue(queueB, B1); + assertNull("Refresh needs to support delete of leaf queue ", queueB3); + + // reset back to default configuration for testing parent queue delete + conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + cs.reinitialize(conf, rmContext); + checkQueueStructureCapacities(cs); + + // set the configurations such that it fails once but should be successfull + // next time + queues = cs.getCapacitySchedulerQueueManager().getShortNameQueues(); + CSQueue bQueue = Mockito.spy((ParentQueue) queues.get("b")); + when(bQueue.getState()).thenReturn(QueueState.DRAINING) + .thenReturn(QueueState.STOPPED); + cs.getCapacitySchedulerQueueManager().addQueue("b", bQueue); + + bQueue = Mockito.spy((LeafQueue) queues.get("b1")); + when(bQueue.getState()).thenReturn(QueueState.STOPPED); + cs.getCapacitySchedulerQueueManager().addQueue("b1", bQueue); + + bQueue = Mockito.spy((LeafQueue) queues.get("b2")); + when(bQueue.getState()).thenReturn(QueueState.STOPPED); + cs.getCapacitySchedulerQueueManager().addQueue("b2", bQueue); + + bQueue = Mockito.spy((LeafQueue) queues.get("b3")); + when(bQueue.getState()).thenReturn(QueueState.STOPPED); + cs.getCapacitySchedulerQueueManager().addQueue("b3", bQueue); + + // test delete Parent queue when there is application running. + conf = new CapacitySchedulerConfiguration(); + setupQueueConfigurationWithoutB(conf); + try { + cs.reinitialize(conf, mockContext); + fail("Expected to throw exception when refresh queue tries to delete a" + + " parent queue with running apps in children queue"); + } catch (IOException e) { + // ignore + } + + // test delete Parent queue when there is no application running. + conf = new CapacitySchedulerConfiguration(); + setupQueueConfigurationWithoutB(conf); + try { + cs.reinitialize(conf, mockContext); + } catch (IOException e) { + fail("Expected to not throw exception when refresh queue tries to delete" + + " a queue without running apps"); + } + rootQueue = cs.getRootQueue(); + queueB = findQueue(rootQueue, B); + String message = + "Refresh needs to support delete of Parent queue and its children."; + assertNull(message, queueB); + assertNull(message, + cs.getCapacitySchedulerQueueManager().getQueues().get("b")); + assertNull(message, + cs.getCapacitySchedulerQueueManager().getQueues().get("b1")); + assertNull(message, + cs.getCapacitySchedulerQueueManager().getQueues().get("b2")); + + cs.stop(); + } + + /** + * Test for all child queue deletion and thus making parent queue a child. + * + * @throws Exception + */ + @Test + public void testRefreshQueuesWithAllChildQueuesDeleted() throws Exception { + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new NMTokenSecretManagerInRM(conf), + new ClientToAMTokenSecretManagerInRM(), null); + setupQueueConfiguration(conf); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + cs.init(conf); + cs.start(); + cs.reinitialize(conf, rmContext); + checkQueueStructureCapacities(cs); + + // test delete all leaf queues when there is no application running. + Map queues = + cs.getCapacitySchedulerQueueManager().getShortNameQueues(); + + CSQueue bQueue = Mockito.spy((LeafQueue) queues.get("b1")); + when(bQueue.getState()).thenReturn(QueueState.RUNNING) + .thenReturn(QueueState.STOPPED); + cs.getCapacitySchedulerQueueManager().addQueue("b1", bQueue); + + bQueue = Mockito.spy((LeafQueue) queues.get("b2")); + when(bQueue.getState()).thenReturn(QueueState.STOPPED); + cs.getCapacitySchedulerQueueManager().addQueue("b2", bQueue); + + bQueue = Mockito.spy((LeafQueue) queues.get("b3")); + when(bQueue.getState()).thenReturn(QueueState.STOPPED); + cs.getCapacitySchedulerQueueManager().addQueue("b3", bQueue); + + conf = new CapacitySchedulerConfiguration(); + setupQueueConfWithoutChildrenOfB(conf); + + // test convert parent queue to leaf queue(root.b) when there is no + // application running. + try { + cs.reinitialize(conf, mockContext); + fail("Expected to throw exception when refresh queue tries to make parent" + + " queue a child queue when one of its children is still running."); + } catch (IOException e) { + //do not do anything, expected exception + } + + // test delete leaf queues(root.b.b1,b2,b3) when there is no application + // running. + try { + cs.reinitialize(conf, mockContext); + } catch (IOException e) { + e.printStackTrace(); + fail("Expected to NOT throw exception when refresh queue tries to delete" + + " all children of a parent queue(without running apps)."); + } + CSQueue rootQueue = cs.getRootQueue(); + CSQueue queueB = findQueue(rootQueue, B); + assertNotNull("Parent Queue B should not be deleted", queueB); + Assert.assertTrue("As Queue'B children are not deleted", + queueB instanceof LeafQueue); + + String message = + "Refresh needs to support delete of all children of Parent queue."; + assertNull(message, + cs.getCapacitySchedulerQueueManager().getQueues().get("b3")); + assertNull(message, + cs.getCapacitySchedulerQueueManager().getQueues().get("b1")); + assertNull(message, + cs.getCapacitySchedulerQueueManager().getQueues().get("b2")); + + cs.stop(); + } + + /** + * Test if we can convert a leaf queue to a parent queue. + * + * @throws Exception + */ + @Test(timeout = 10000) + public void testConvertLeafQueueToParentQueue() throws Exception { + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new NMTokenSecretManagerInRM(conf), + new ClientToAMTokenSecretManagerInRM(), null); + setupQueueConfiguration(conf); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + cs.init(conf); + cs.start(); + cs.reinitialize(conf, rmContext); + checkQueueStructureCapacities(cs); + + String targetQueue = "b1"; + CSQueue b1 = cs.getQueue(targetQueue); + Assert.assertEquals(QueueState.RUNNING, b1.getState()); + + // test if we can convert a leaf queue which is in RUNNING state + conf = new CapacitySchedulerConfiguration(); + setupQueueConfigurationWithB1AsParentQueue(conf); + try { + cs.reinitialize(conf, mockContext); + fail("Expected to throw exception when refresh queue tries to convert" + + " a child queue to a parent queue."); + } catch (IOException e) { + // ignore + } + + // now set queue state for b1 to STOPPED + conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + conf.set("yarn.scheduler.capacity.root.b.b1.state", "STOPPED"); + cs.reinitialize(conf, mockContext); + Assert.assertEquals(QueueState.STOPPED, b1.getState()); + + // test if we can convert a leaf queue which is in STOPPED state + conf = new CapacitySchedulerConfiguration(); + setupQueueConfigurationWithB1AsParentQueue(conf); + try { + cs.reinitialize(conf, mockContext); + } catch (IOException e) { + fail("Expected to NOT throw exception when refresh queue tries" + + " to convert a leaf queue WITHOUT running apps"); + } + b1 = cs.getQueue(targetQueue); + Assert.assertTrue(b1 instanceof ParentQueue); + Assert.assertEquals(QueueState.RUNNING, b1.getState()); + Assert.assertTrue(!b1.getChildQueues().isEmpty()); + } + + @Test + public void testQueuesMaxAllocationInheritance() throws Exception { + // queue level max allocation is set by the queue configuration explicitly + // or inherits from the parent. + + CapacityScheduler cs = new CapacityScheduler(); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + setMaxAllocMb(conf, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); + setMaxAllocVcores(conf, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + + // Test the child queue overrides + setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT, + "memory-mb=4096,vcores=2"); + setMaxAllocation(conf, A1, "memory-mb=6144,vcores=2"); + setMaxAllocation(conf, B, "memory-mb=5120, vcores=2"); + setMaxAllocation(conf, B2, "memory-mb=1024, vcores=2"); + + cs.init(conf); + cs.start(); + cs.reinitialize(conf, mockContext); + checkQueueStructureCapacities(cs); + + CSQueue rootQueue = cs.getRootQueue(); + CSQueue queueA = findQueue(rootQueue, A); + CSQueue queueB = findQueue(rootQueue, B); + CSQueue queueA1 = findQueue(queueA, A1); + CSQueue queueA2 = findQueue(queueA, A2); + CSQueue queueB1 = findQueue(queueB, B1); + CSQueue queueB2 = findQueue(queueB, B2); + + assertEquals("max capability MB in CS", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumResourceCapability().getMemorySize()); + assertEquals("max capability vcores in CS", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + cs.getMaximumResourceCapability().getVirtualCores()); + assertEquals("max allocation MB A1", + 6144, + queueA1.getMaximumAllocation().getMemorySize()); + assertEquals("max allocation vcores A1", + 2, + queueA1.getMaximumAllocation().getVirtualCores()); + assertEquals("max allocation MB A2", 4096, + queueA2.getMaximumAllocation().getMemorySize()); + assertEquals("max allocation vcores A2", + 2, + queueA2.getMaximumAllocation().getVirtualCores()); + assertEquals("max allocation MB B", 5120, + queueB.getMaximumAllocation().getMemorySize()); + assertEquals("max allocation MB B1", 5120, + queueB1.getMaximumAllocation().getMemorySize()); + assertEquals("max allocation MB B2", 1024, + queueB2.getMaximumAllocation().getMemorySize()); + + // Test get the max-allocation from different parent + unsetMaxAllocation(conf, A1); + unsetMaxAllocation(conf, B); + unsetMaxAllocation(conf, B1); + setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT, + "memory-mb=6144,vcores=2"); + setMaxAllocation(conf, A, "memory-mb=8192,vcores=2"); + + cs.reinitialize(conf, mockContext); + + assertEquals("max capability MB in CS", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumResourceCapability().getMemorySize()); + assertEquals("max capability vcores in CS", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + cs.getMaximumResourceCapability().getVirtualCores()); + assertEquals("max allocation MB A1", + 8192, + queueA1.getMaximumAllocation().getMemorySize()); + assertEquals("max allocation vcores A1", + 2, + queueA1.getMaximumAllocation().getVirtualCores()); + assertEquals("max allocation MB B1", + 6144, + queueB1.getMaximumAllocation().getMemorySize()); + assertEquals("max allocation vcores B1", + 2, + queueB1.getMaximumAllocation().getVirtualCores()); + + // Test the default + unsetMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT); + unsetMaxAllocation(conf, A); + unsetMaxAllocation(conf, A1); + cs.reinitialize(conf, mockContext); + + assertEquals("max capability MB in CS", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumResourceCapability().getMemorySize()); + assertEquals("max capability vcores in CS", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + cs.getMaximumResourceCapability().getVirtualCores()); + assertEquals("max allocation MB A1", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + queueA1.getMaximumAllocation().getMemorySize()); + assertEquals("max allocation vcores A1", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + queueA1.getMaximumAllocation().getVirtualCores()); + assertEquals("max allocation MB A2", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + queueA2.getMaximumAllocation().getMemorySize()); + assertEquals("max allocation vcores A2", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + queueA2.getMaximumAllocation().getVirtualCores()); + } + + @Test + public void testVerifyQueuesMaxAllocationConf() throws Exception { + // queue level max allocation can't exceed the cluster setting + + CapacityScheduler cs = new CapacityScheduler(); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + setMaxAllocMb(conf, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); + setMaxAllocVcores(conf, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + + long largerMem = + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 1024; + long largerVcores = + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 10; + + cs.init(conf); + cs.start(); + cs.reinitialize(conf, mockContext); + checkQueueStructureCapacities(cs); + + setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT, + "memory-mb=" + largerMem + ",vcores=2"); + try { + cs.reinitialize(conf, mockContext); + fail("Queue Root maximum allocation can't exceed the cluster setting"); + } catch (Exception e) { + assertTrue("maximum allocation exception", + e.getCause().getMessage().contains("maximum allocation")); + } + + setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT, + "memory-mb=4096,vcores=2"); + setMaxAllocation(conf, A, "memory-mb=6144,vcores=2"); + setMaxAllocation(conf, A1, "memory-mb=" + largerMem + ",vcores=2"); + try { + cs.reinitialize(conf, mockContext); + fail("Queue A1 maximum allocation can't exceed the cluster setting"); + } catch (Exception e) { + assertTrue("maximum allocation exception", + e.getCause().getMessage().contains("maximum allocation")); + } + setMaxAllocation(conf, A1, "memory-mb=8192" + ",vcores=" + largerVcores); + try { + cs.reinitialize(conf, mockContext); + fail("Queue A1 maximum allocation can't exceed the cluster setting"); + } catch (Exception e) { + assertTrue("maximum allocation exception", + e.getCause().getMessage().contains("maximum allocation")); + } + } +} From 4d54a0ef92cfc9f28376766635abe2c5eb504fb2 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth <954799+szilard-nemeth@users.noreply.github.com> Date: Fri, 17 Dec 2021 00:18:14 +0100 Subject: [PATCH 25/33] YARN-10951. CapacityScheduler: Move all fields and initializer code that belongs to async scheduling to a new class (#3800). Contributed by Szilard Nemeth --- .../scheduler/capacity/CapacityScheduler.java | 200 +++++++++++------- .../CapacitySchedulerConfiguration.java | 6 + .../TestCapacitySchedulerAsyncScheduling.java | 2 +- .../capacity/TestCapacitySchedulerPerf.java | 4 +- .../scheduler/capacity/TestLeafQueue.java | 2 +- .../scheduler/capacity/TestReservations.java | 74 +++---- .../scheduler/capacity/TestUtils.java | 11 +- 7 files changed, 182 insertions(+), 117 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index abd40a8062b530..16c18bcd5c3344 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -187,9 +187,6 @@ public class CapacityScheduler extends private WorkflowPriorityMappingsManager workflowPriorityMappingsMgr; - // timeout to join when we stop this service - protected final long THREAD_JOIN_TIMEOUT_MS = 1000; - private PreemptionManager preemptionManager = new PreemptionManager(); private volatile boolean isLazyPreemptionEnabled = false; @@ -227,10 +224,7 @@ public Configuration getConf() { private ResourceCalculator calculator; private boolean usePortForNodeName; - private boolean scheduleAsynchronously; - @VisibleForTesting - protected List asyncSchedulerThreads; - private ResourceCommitterService resourceCommitterService; + private AsyncSchedulingConfiguration asyncSchedulingConf; private RMNodeLabelsManager labelManager; private AppPriorityACLsManager appPriorityACLManager; private boolean multiNodePlacementEnabled; @@ -238,16 +232,6 @@ public Configuration getConf() { private boolean printedVerboseLoggingForAsyncScheduling; private boolean appShouldFailFast; - /** - * EXPERT - */ - private long asyncScheduleInterval; - private static final String ASYNC_SCHEDULER_INTERVAL = - CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX - + ".scheduling-interval-ms"; - private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5; - private long asyncMaxPendingBacklogs; - private CSMaxRunningAppsEnforcer maxRunningEnforcer; public CapacityScheduler() { @@ -376,27 +360,7 @@ private ResourceCalculator initResourceCalculator() { } private void initAsyncSchedulingProperties() { - scheduleAsynchronously = this.conf.getScheduleAynschronously(); - asyncScheduleInterval = this.conf.getLong(ASYNC_SCHEDULER_INTERVAL, - DEFAULT_ASYNC_SCHEDULER_INTERVAL); - - // number of threads for async scheduling - int maxAsyncSchedulingThreads = this.conf.getInt( - CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD, 1); - maxAsyncSchedulingThreads = Math.max(maxAsyncSchedulingThreads, 1); - - if (scheduleAsynchronously) { - asyncSchedulerThreads = new ArrayList<>(); - for (int i = 0; i < maxAsyncSchedulingThreads; i++) { - asyncSchedulerThreads.add(new AsyncScheduleThread(this)); - } - resourceCommitterService = new ResourceCommitterService(this); - asyncMaxPendingBacklogs = this.conf.getInt( - CapacitySchedulerConfiguration. - SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS, - CapacitySchedulerConfiguration. - DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS); - } + this.asyncSchedulingConf = new AsyncSchedulingConfiguration(conf, this); } private void initMultiNodePlacement() { @@ -419,8 +383,8 @@ private void printSchedulerInitialized() { getResourceCalculator().getClass(), getMinimumResourceCapability(), getMaximumResourceCapability(), - scheduleAsynchronously, - asyncScheduleInterval, + asyncSchedulingConf.isScheduleAsynchronously(), + asyncSchedulingConf.getAsyncScheduleInterval(), multiNodePlacementEnabled, assignMultipleEnabled, maxAssignPerHeartbeat, @@ -431,15 +395,7 @@ private void startSchedulerThreads() { writeLock.lock(); try { activitiesManager.start(); - if (scheduleAsynchronously) { - Preconditions.checkNotNull(asyncSchedulerThreads, - "asyncSchedulerThreads is null"); - for (Thread t : asyncSchedulerThreads) { - t.start(); - } - - resourceCommitterService.start(); - } + asyncSchedulingConf.startThreads(); } finally { writeLock.unlock(); } @@ -465,14 +421,7 @@ public void serviceStop() throws Exception { writeLock.lock(); try { this.activitiesManager.stop(); - if (scheduleAsynchronously && asyncSchedulerThreads != null) { - for (Thread t : asyncSchedulerThreads) { - t.interrupt(); - t.join(THREAD_JOIN_TIMEOUT_MS); - } - resourceCommitterService.interrupt(); - resourceCommitterService.join(THREAD_JOIN_TIMEOUT_MS); - } + asyncSchedulingConf.serviceStopInvoked(); } finally { writeLock.unlock(); } @@ -539,7 +488,7 @@ public void reinitialize(Configuration newConf, RMContext rmContext) } long getAsyncScheduleInterval() { - return asyncScheduleInterval; + return asyncSchedulingConf.getAsyncScheduleInterval(); } private final static Random random = new Random(System.currentTimeMillis()); @@ -671,6 +620,11 @@ static void schedule(CapacityScheduler cs) throws InterruptedException{ Thread.sleep(cs.getAsyncScheduleInterval()); } + @VisibleForTesting + public void setAsyncSchedulingConf(AsyncSchedulingConfiguration conf) { + this.asyncSchedulingConf = conf; + } + static class AsyncScheduleThread extends Thread { private final CapacityScheduler cs; @@ -692,7 +646,7 @@ public void run() { } else { // Don't run schedule if we have some pending backlogs already if (cs.getAsyncSchedulingPendingBacklogs() - > cs.asyncMaxPendingBacklogs) { + > cs.asyncSchedulingConf.getAsyncMaxPendingBacklogs()) { Thread.sleep(1); } else{ schedule(cs); @@ -1479,7 +1433,7 @@ protected void nodeUpdate(RMNode rmNode) { } // Try to do scheduling - if (!scheduleAsynchronously) { + if (!asyncSchedulingConf.isScheduleAsynchronously()) { writeLock.lock(); try { // reset allocation and reservation stats before we start doing any @@ -2291,8 +2245,8 @@ private void addNode(RMNode nodeManager) { "Added node " + nodeManager.getNodeAddress() + " clusterResource: " + clusterResource); - if (scheduleAsynchronously && getNumClusterNodes() == 1) { - for (AsyncScheduleThread t : asyncSchedulerThreads) { + if (asyncSchedulingConf.isScheduleAsynchronously() && getNumClusterNodes() == 1) { + for (AsyncScheduleThread t : asyncSchedulingConf.asyncSchedulerThreads) { t.beginSchedule(); } } @@ -2340,11 +2294,7 @@ private void removeNode(RMNode nodeInfo) { new ResourceLimits(clusterResource)); int numNodes = nodeTracker.nodeCount(); - if (scheduleAsynchronously && numNodes == 0) { - for (AsyncScheduleThread t : asyncSchedulerThreads) { - t.suspendSchedule(); - } - } + asyncSchedulingConf.nodeRemoved(numNodes); LOG.info( "Removed node " + nodeInfo.getNodeAddress() + " clusterResource: " @@ -3092,9 +3042,9 @@ public void submitResourceCommitRequest(Resource cluster, return; } - if (scheduleAsynchronously) { + if (asyncSchedulingConf.isScheduleAsynchronously()) { // Submit to a commit thread and commit it async-ly - resourceCommitterService.addNewCommitRequest(request); + asyncSchedulingConf.resourceCommitterService.addNewCommitRequest(request); } else{ // Otherwise do it sync-ly. tryCommit(cluster, request, true); @@ -3339,10 +3289,7 @@ public boolean tryCommit(Resource cluster, ResourceCommitRequest r, } public int getAsyncSchedulingPendingBacklogs() { - if (scheduleAsynchronously) { - return resourceCommitterService.getPendingBacklogs(); - } - return 0; + return asyncSchedulingConf.getPendingBacklogs(); } @Override @@ -3483,7 +3430,7 @@ public boolean isMultiNodePlacementEnabled() { } public int getNumAsyncSchedulerThreads() { - return asyncSchedulerThreads == null ? 0 : asyncSchedulerThreads.size(); + return asyncSchedulingConf.getNumAsyncSchedulerThreads(); } @VisibleForTesting @@ -3503,4 +3450,109 @@ public boolean placementConstraintEnabled() { public void setQueueManager(CapacitySchedulerQueueManager qm) { this.queueManager = qm; } + + @VisibleForTesting + public List getAsyncSchedulerThreads() { + return asyncSchedulingConf.getAsyncSchedulerThreads(); + } + + static class AsyncSchedulingConfiguration { + // timeout to join when we stop this service + private static final long THREAD_JOIN_TIMEOUT_MS = 1000; + + @VisibleForTesting + protected List asyncSchedulerThreads; + private ResourceCommitterService resourceCommitterService; + + private long asyncScheduleInterval; + private static final String ASYNC_SCHEDULER_INTERVAL = + CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX + + ".scheduling-interval-ms"; + private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5; + private long asyncMaxPendingBacklogs; + + private final boolean scheduleAsynchronously; + + AsyncSchedulingConfiguration(CapacitySchedulerConfiguration conf, + CapacityScheduler cs) { + this.scheduleAsynchronously = conf.getScheduleAynschronously(); + if (this.scheduleAsynchronously) { + this.asyncScheduleInterval = conf.getLong( + CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_INTERVAL, + CapacitySchedulerConfiguration.DEFAULT_SCHEDULE_ASYNCHRONOUSLY_INTERVAL); + // number of threads for async scheduling + int maxAsyncSchedulingThreads = conf.getInt( + CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD, + 1); + maxAsyncSchedulingThreads = Math.max(maxAsyncSchedulingThreads, 1); + this.asyncMaxPendingBacklogs = conf.getInt( + CapacitySchedulerConfiguration. + SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS, + CapacitySchedulerConfiguration. + DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS); + + this.asyncSchedulerThreads = new ArrayList<>(); + for (int i = 0; i < maxAsyncSchedulingThreads; i++) { + asyncSchedulerThreads.add(new AsyncScheduleThread(cs)); + } + this.resourceCommitterService = new ResourceCommitterService(cs); + } + } + public boolean isScheduleAsynchronously() { + return scheduleAsynchronously; + } + public long getAsyncScheduleInterval() { + return asyncScheduleInterval; + } + public long getAsyncMaxPendingBacklogs() { + return asyncMaxPendingBacklogs; + } + + public void startThreads() { + if (scheduleAsynchronously) { + Preconditions.checkNotNull(asyncSchedulerThreads, + "asyncSchedulerThreads is null"); + for (Thread t : asyncSchedulerThreads) { + t.start(); + } + + resourceCommitterService.start(); + } + } + + public void serviceStopInvoked() throws InterruptedException { + if (scheduleAsynchronously && asyncSchedulerThreads != null) { + for (Thread t : asyncSchedulerThreads) { + t.interrupt(); + t.join(THREAD_JOIN_TIMEOUT_MS); + } + resourceCommitterService.interrupt(); + resourceCommitterService.join(THREAD_JOIN_TIMEOUT_MS); + } + } + + public void nodeRemoved(int numNodes) { + if (scheduleAsynchronously && numNodes == 0) { + for (AsyncScheduleThread t : asyncSchedulerThreads) { + t.suspendSchedule(); + } + } + } + + public int getPendingBacklogs() { + if (scheduleAsynchronously) { + return resourceCommitterService.getPendingBacklogs(); + } + return 0; + } + + public int getNumAsyncSchedulerThreads() { + return asyncSchedulerThreads == null ? 0 : asyncSchedulerThreads.size(); + } + + @VisibleForTesting + public List getAsyncSchedulerThreads() { + return asyncSchedulerThreads; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 2716ddebbdc6db..628c58576e68da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -273,6 +273,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS = SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-pending-backlogs"; + @Private + public static final String SCHEDULE_ASYNCHRONOUSLY_INTERVAL = + SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".scheduling-interval-ms"; + @Private + public static final long DEFAULT_SCHEDULE_ASYNCHRONOUSLY_INTERVAL = 5; + @Private public static final String APP_FAIL_FAST = PREFIX + "application.fail-fast"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java index 98214a030c954c..b36e0edc735037 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java @@ -153,7 +153,7 @@ public RMNodeLabelsManager createNodeLabelManager() { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); for (CapacityScheduler.AsyncScheduleThread thread : - cs.asyncSchedulerThreads) { + cs.getAsyncSchedulerThreads()) { Assert.assertTrue(thread.getName() .startsWith("AsyncCapacitySchedulerThread")); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java index b71fe063927ac8..b8209a54952e78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java @@ -237,7 +237,7 @@ private void testUserLimitThroughputWithNumberOfResourceTypes( if (numThreads > 0) { // disable async scheduling threads - for (CapacityScheduler.AsyncScheduleThread t : cs.asyncSchedulerThreads) { + for (CapacityScheduler.AsyncScheduleThread t : cs.getAsyncSchedulerThreads()) { t.suspendSchedule(); } } @@ -268,7 +268,7 @@ private void testUserLimitThroughputWithNumberOfResourceTypes( if (numThreads > 0) { // enable async scheduling threads - for (CapacityScheduler.AsyncScheduleThread t : cs.asyncSchedulerThreads) { + for (CapacityScheduler.AsyncScheduleThread t : cs.getAsyncSchedulerThreads()) { t.beginSchedule(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 1da7ce18ee01cb..eca065b1487664 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -936,7 +936,7 @@ private void applyCSAssignment(Resource clusterResource, CSAssignment assign, LeafQueue q, final Map nodes, final Map apps) throws IOException { - TestUtils.applyResourceCommitRequest(clusterResource, assign, nodes, apps); + TestUtils.applyResourceCommitRequest(clusterResource, assign, nodes, apps, csConf); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index 53b1d160dc2898..c6f947febc00ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -291,7 +291,7 @@ public void testReservation() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(2 * GB, a.getUsedResources().getMemorySize()); assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -305,7 +305,7 @@ public void testReservation() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(5 * GB, a.getUsedResources().getMemorySize()); assertEquals(5 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -319,7 +319,7 @@ public void testReservation() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_1, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(8 * GB, a.getUsedResources().getMemorySize()); assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -337,7 +337,7 @@ public void testReservation() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(13 * GB, a.getUsedResources().getMemorySize()); assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(5 * GB, a.getMetrics().getReservedMB()); @@ -356,7 +356,7 @@ public void testReservation() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_2, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(18 * GB, a.getUsedResources().getMemorySize()); assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(5 * GB, a.getMetrics().getReservedMB()); @@ -376,7 +376,7 @@ public void testReservation() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_1, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(18 * GB, a.getUsedResources().getMemorySize()); assertEquals(18 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -477,7 +477,7 @@ public void testReservationLimitOtherUsers() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(2 * GB, a.getUsedResources().getMemorySize()); assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, app_1.getCurrentConsumption().getMemorySize()); @@ -491,7 +491,7 @@ public void testReservationLimitOtherUsers() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_1, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(4 * GB, a.getUsedResources().getMemorySize()); assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(2 * GB, app_1.getCurrentConsumption().getMemorySize()); @@ -514,7 +514,7 @@ public void testReservationLimitOtherUsers() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(12 * GB, a.getUsedResources().getMemorySize()); assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(2 * GB, app_1.getCurrentConsumption().getMemorySize()); @@ -530,7 +530,7 @@ public void testReservationLimitOtherUsers() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_1, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(14 * GB, a.getUsedResources().getMemorySize()); assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(4 * GB, app_1.getCurrentConsumption().getMemorySize()); @@ -628,7 +628,7 @@ public void testReservationNoContinueLook() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(2 * GB, a.getUsedResources().getMemorySize()); assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -642,7 +642,7 @@ public void testReservationNoContinueLook() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(5 * GB, a.getUsedResources().getMemorySize()); assertEquals(5 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -656,7 +656,7 @@ public void testReservationNoContinueLook() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_1, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(8 * GB, a.getUsedResources().getMemorySize()); assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -674,7 +674,7 @@ public void testReservationNoContinueLook() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(13 * GB, a.getUsedResources().getMemorySize()); assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(5 * GB, a.getMetrics().getReservedMB()); @@ -693,7 +693,7 @@ public void testReservationNoContinueLook() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_2, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(18 * GB, a.getUsedResources().getMemorySize()); assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(5 * GB, a.getMetrics().getReservedMB()); @@ -713,7 +713,7 @@ public void testReservationNoContinueLook() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_1, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(18 * GB, a.getUsedResources().getMemorySize()); assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(5 * GB, a.getMetrics().getReservedMB()); @@ -811,7 +811,7 @@ public void testAssignContainersNeedToUnreserve() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(2 * GB, a.getUsedResources().getMemorySize()); assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -824,7 +824,7 @@ public void testAssignContainersNeedToUnreserve() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(5 * GB, a.getUsedResources().getMemorySize()); assertEquals(5 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -837,7 +837,7 @@ public void testAssignContainersNeedToUnreserve() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_1, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(8 * GB, a.getUsedResources().getMemorySize()); assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -854,7 +854,7 @@ public void testAssignContainersNeedToUnreserve() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(13 * GB, a.getUsedResources().getMemorySize()); assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(5 * GB, a.getMetrics().getReservedMB()); @@ -872,7 +872,7 @@ public void testAssignContainersNeedToUnreserve() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_1, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(13 * GB, a.getUsedResources().getMemorySize()); assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -1102,7 +1102,7 @@ public void testAssignToQueue() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(2 * GB, a.getUsedResources().getMemorySize()); assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -1115,7 +1115,7 @@ public void testAssignToQueue() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(5 * GB, a.getUsedResources().getMemorySize()); assertEquals(5 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -1128,7 +1128,7 @@ public void testAssignToQueue() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_1, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(8 * GB, a.getUsedResources().getMemorySize()); assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -1144,7 +1144,7 @@ public void testAssignToQueue() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(13 * GB, a.getUsedResources().getMemorySize()); assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(5 * GB, a.getMetrics().getReservedMB()); @@ -1293,7 +1293,7 @@ public void testAssignToUser() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(2 * GB, a.getUsedResources().getMemorySize()); assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -1306,7 +1306,7 @@ public void testAssignToUser() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(5 * GB, a.getUsedResources().getMemorySize()); assertEquals(5 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -1319,7 +1319,7 @@ public void testAssignToUser() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_1, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(8 * GB, a.getUsedResources().getMemorySize()); assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -1335,7 +1335,7 @@ public void testAssignToUser() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(13 * GB, a.getUsedResources().getMemorySize()); assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(5 * GB, app_0.getCurrentReservation().getMemorySize()); @@ -1462,7 +1462,7 @@ public void testReservationsNoneAvailable() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(2 * GB, a.getUsedResources().getMemorySize()); assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -1476,7 +1476,7 @@ public void testReservationsNoneAvailable() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(5 * GB, a.getUsedResources().getMemorySize()); assertEquals(5 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -1490,7 +1490,7 @@ public void testReservationsNoneAvailable() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_1, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(8 * GB, a.getUsedResources().getMemorySize()); assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -1508,7 +1508,7 @@ public void testReservationsNoneAvailable() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(Resources.createResource(10 * GB)), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(8 * GB, a.getUsedResources().getMemorySize()); assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -1526,7 +1526,7 @@ public void testReservationsNoneAvailable() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_2, new ResourceLimits(Resources.createResource(10 * GB)), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(8 * GB, a.getUsedResources().getMemorySize()); assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -1542,7 +1542,7 @@ public void testReservationsNoneAvailable() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_2, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(13 * GB, a.getUsedResources().getMemorySize()); assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(0 * GB, a.getMetrics().getReservedMB()); @@ -1557,7 +1557,7 @@ public void testReservationsNoneAvailable() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(21 * GB, a.getUsedResources().getMemorySize()); assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(8 * GB, a.getMetrics().getReservedMB()); @@ -1574,7 +1574,7 @@ public void testReservationsNoneAvailable() throws Exception { TestUtils.applyResourceCommitRequest(clusterResource, a.assignContainers(clusterResource, node_2, new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps); + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf); assertEquals(21 * GB, a.getUsedResources().getMemorySize()); assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(8 * GB, a.getMetrics().getReservedMB()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index 026206ac38f47f..28ca66847de006 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -447,7 +447,14 @@ public static SchedulerRequestKey toSchedulerKey(Priority pri, public static void applyResourceCommitRequest(Resource clusterResource, CSAssignment csAssignment, final Map nodes, - final Map apps) + final Map apps) throws IOException { + applyResourceCommitRequest(clusterResource, csAssignment, nodes, apps, null); + } + + public static void applyResourceCommitRequest(Resource clusterResource, + CSAssignment csAssignment, + final Map nodes, + final Map apps, CapacitySchedulerConfiguration csConf) throws IOException { CapacityScheduler cs = new CapacityScheduler() { @Override @@ -461,7 +468,7 @@ public FiCaSchedulerApp getApplicationAttempt( return apps.get(applicationAttemptId); } }; - + cs.setAsyncSchedulingConf(new CapacityScheduler.AsyncSchedulingConfiguration(csConf, cs)); cs.setResourceCalculator(new DefaultResourceCalculator()); cs.submitResourceCommitRequest(clusterResource, From a8635b87540854def598f7c1d9ab7400dc499f49 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth <954799+szilard-nemeth@users.noreply.github.com> Date: Fri, 17 Dec 2021 00:34:16 +0100 Subject: [PATCH 26/33] YARN-10427. Duplicate Job IDs in SLS output (#3809). Contributed by Szilard Nemeth --- .../hadoop/yarn/sls/appmaster/AMSimulator.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java index 5315eaa1a31187..922f9a2b97a6c7 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java @@ -75,6 +75,7 @@ @Private @Unstable public abstract class AMSimulator extends TaskRunner.Task { + private static final long FINISH_TIME_NOT_INITIALIZED = Long.MIN_VALUE; // resource manager protected ResourceManager rm; // main @@ -102,7 +103,7 @@ public abstract class AMSimulator extends TaskRunner.Task { protected long traceStartTimeMS; protected long traceFinishTimeMS; protected long simulateStartTimeMS; - protected long simulateFinishTimeMS; + protected long simulateFinishTimeMS = FINISH_TIME_NOT_INITIALIZED; // whether tracked in Metrics protected boolean isTracked; // progress @@ -226,6 +227,16 @@ public void middleStep() throws Exception { @Override public void lastStep() throws Exception { + if (simulateFinishTimeMS != FINISH_TIME_NOT_INITIALIZED) { + // The finish time is already recorded. + // Different value from zero means lastStep was called before. + // We want to prevent lastStep to be called more than once. + // See YARN-10427 for more details. + LOG.warn("Method AMSimulator#lastStep was already called. " + + "Skipping execution of method for application: {}", appId); + return; + } + LOG.info("Application {} is shutting down.", appId); // unregister tracking if (isTracked) { From 057297b9550bcb76de94adea58772d518aed2fe4 Mon Sep 17 00:00:00 2001 From: liubingxing <1476659627@qq.com> Date: Thu, 16 Dec 2021 23:31:28 -0600 Subject: [PATCH 27/33] HDFS-16352. return the real datanode numBlocks in #getDatanodeStorageReport (#3714). Contributed by liubingxing. Signed-off-by: He Xiaoqiao --- .../hadoop/hdfs/protocol/DatanodeInfo.java | 6 ++-- .../blockmanagement/DatanodeManager.java | 3 +- .../TestNameNodeRpcServerMethods.java | 30 +++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java index bba90a05794ba5..fbe6bcc4629d8f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java @@ -698,9 +698,10 @@ public static class DatanodeInfoBuilder { private long nonDfsUsed = 0L; private long lastBlockReportTime = 0L; private long lastBlockReportMonotonic = 0L; - private int numBlocks; - + private int numBlocks = 0; + // Please use setNumBlocks explicitly to set numBlocks as this method doesn't have + // sufficient info about numBlocks public DatanodeInfoBuilder setFrom(DatanodeInfo from) { this.capacity = from.getCapacity(); this.dfsUsed = from.getDfsUsed(); @@ -717,7 +718,6 @@ public DatanodeInfoBuilder setFrom(DatanodeInfo from) { this.upgradeDomain = from.getUpgradeDomain(); this.lastBlockReportTime = from.getLastBlockReportTime(); this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic(); - this.numBlocks = from.getNumBlocks(); setNodeID(from); return this; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index ef51c6ca074ab8..cfb1d83ec5bc69 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -2182,7 +2182,8 @@ public DatanodeStorageReport[] getDatanodeStorageReport( for (int i = 0; i < reports.length; i++) { final DatanodeDescriptor d = datanodes.get(i); reports[i] = new DatanodeStorageReport( - new DatanodeInfoBuilder().setFrom(d).build(), d.getStorageReports()); + new DatanodeInfoBuilder().setFrom(d).setNumBlocks(d.numBlocks()).build(), + d.getStorageReports()); } return reports; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServerMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServerMethods.java index a32e2188e88224..50740bd06e8313 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServerMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServerMethods.java @@ -20,9 +20,14 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.test.GenericTestUtils; @@ -31,6 +36,8 @@ import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertEquals; + public class TestNameNodeRpcServerMethods { private static NamenodeProtocols nnRpc; private static Configuration conf; @@ -83,4 +90,27 @@ public void testDeleteSnapshotWhenSnapshotNameIsEmpty() throws Exception { } + @Test + public void testGetDatanodeStorageReportWithNumBLocksNotZero() throws Exception { + int buffSize = 1024; + long blockSize = 1024 * 1024; + String file = "/testFile"; + DistributedFileSystem dfs = cluster.getFileSystem(); + FSDataOutputStream outputStream = dfs.create( + new Path(file), true, buffSize, (short)1, blockSize); + byte[] outBuffer = new byte[buffSize]; + for (int i = 0; i < buffSize; i++) { + outBuffer[i] = (byte) (i & 0x00ff); + } + outputStream.write(outBuffer); + outputStream.close(); + + int numBlocks = 0; + DatanodeStorageReport[] reports + = nnRpc.getDatanodeStorageReport(HdfsConstants.DatanodeReportType.ALL); + for (DatanodeStorageReport r : reports) { + numBlocks += r.getDatanodeInfo().getNumBlocks(); + } + assertEquals(1, numBlocks); + } } From cd377889c9498b32e5b2026ad435883d12930701 Mon Sep 17 00:00:00 2001 From: Dhananjay Badaya <11253243+dbadaya1@users.noreply.github.com> Date: Fri, 17 Dec 2021 12:35:46 +0530 Subject: [PATCH 28/33] HADOOP-13500. Synchronizing iteration of Configuration properties object (#3775) Signed-off-by: Akira Ajisaka --- .../org/apache/hadoop/conf/Configuration.java | 10 ++++--- .../apache/hadoop/conf/TestConfiguration.java | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 28be4beb51eb11..1f809b7b547064 100755 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -2978,11 +2978,13 @@ public Iterator> iterator() { // methods that allow non-strings to be put into configurations are removed, // we could replace properties with a Map and get rid of this // code. - Map result = new HashMap(); - for(Map.Entry item: getProps().entrySet()) { - if (item.getKey() instanceof String && - item.getValue() instanceof String) { + Properties props = getProps(); + Map result = new HashMap<>(); + synchronized (props) { + for (Map.Entry item : props.entrySet()) { + if (item.getKey() instanceof String && item.getValue() instanceof String) { result.put((String) item.getKey(), (String) item.getValue()); + } } } return result.entrySet().iterator(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 731fbc430b7465..b3487ef309fc93 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.ConcurrentModificationException; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -45,6 +46,7 @@ import java.util.Properties; import java.util.Random; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; import static java.util.concurrent.TimeUnit.*; @@ -2687,4 +2689,31 @@ private static Configuration checkCDATA(byte[] bytes) { assertEquals(" prefix >cdata\nsuffix ", conf.get("cdata-whitespace")); return conf; } + + @Test + public void testConcurrentModificationDuringIteration() throws InterruptedException { + Configuration configuration = new Configuration(); + new Thread(() -> { + while (true) { + configuration.set(String.valueOf(Math.random()), String.valueOf(Math.random())); + } + }).start(); + + AtomicBoolean exceptionOccurred = new AtomicBoolean(false); + + new Thread(() -> { + while (true) { + try { + configuration.iterator(); + } catch (final ConcurrentModificationException e) { + exceptionOccurred.set(true); + break; + } + } + }).start(); + + Thread.sleep(1000); //give enough time for threads to run + + assertFalse("ConcurrentModificationException occurred", exceptionOccurred.get()); + } } From 8c306b5fe2b24406ddb3169a690496ecd2639230 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth <954799+szilard-nemeth@users.noreply.github.com> Date: Sat, 18 Dec 2021 04:52:03 +0100 Subject: [PATCH 29/33] YARN-11050 (#3805) --- .../hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java | 2 +- .../server/resourcemanager/webapp/RMWebServiceProtocol.java | 2 +- .../yarn/server/resourcemanager/webapp/RMWebServices.java | 2 +- .../server/router/webapp/DefaultRequestInterceptorREST.java | 2 +- .../yarn/server/router/webapp/FederationInterceptorREST.java | 2 +- .../hadoop/yarn/server/router/webapp/RouterWebServices.java | 4 ++-- .../yarn/server/router/webapp/BaseRouterWebServicesTest.java | 4 ++-- .../yarn/server/router/webapp/MockRESTRequestInterceptor.java | 2 +- .../router/webapp/PassThroughRESTRequestInterceptor.java | 4 ++-- .../yarn/server/router/webapp/TestRouterWebServices.java | 2 +- .../yarn/server/router/webapp/TestRouterWebServicesREST.java | 4 ++-- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java index 82ceed37c2d5d5..791e34ad3d182d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java @@ -126,7 +126,7 @@ public final class RMWSConsts { /** Path for {@code RMWebServiceProtocol#addToClusterNodeLabels}. */ public static final String ADD_NODE_LABELS = "/add-node-labels"; - /** Path for {@code RMWebServiceProtocol#removeFromCluserNodeLabels}. */ + /** Path for {@code RMWebServiceProtocol#removeFromClusterNodeLabels}. */ public static final String REMOVE_NODE_LABELS = "/remove-node-labels"; /** Path for {@code RMWebServiceProtocol#getLabelsOnNode}. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java index f2736e3773c1b3..41fc4ea8709854 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java @@ -398,7 +398,7 @@ Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels, * @return Response containing the status code * @throws Exception in case of bad request */ - Response removeFromCluserNodeLabels(Set oldNodeLabels, + Response removeFromClusterNodeLabels(Set oldNodeLabels, HttpServletRequest hsr) throws Exception; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 041b37c616f041..1dac043ae16b6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -1435,7 +1435,7 @@ public Response addToClusterNodeLabels(final NodeLabelsInfo newNodeLabels, @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) @Override - public Response removeFromCluserNodeLabels( + public Response removeFromClusterNodeLabels( @QueryParam(RMWSConsts.LABELS) Set oldNodeLabels, @Context HttpServletRequest hsr) throws Exception { UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java index 2675b38244016f..21fd2be8546189 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java @@ -330,7 +330,7 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels, } @Override - public Response removeFromCluserNodeLabels(Set oldNodeLabels, + public Response removeFromClusterNodeLabels(Set oldNodeLabels, HttpServletRequest hsr) throws Exception { // oldNodeLabels is specified inside hsr return RouterWebServiceUtil.genericForward(webAppAddress, hsr, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java index 14e062ad4e9852..db2a8edcb2cdbd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java @@ -1205,7 +1205,7 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels, } @Override - public Response removeFromCluserNodeLabels(Set oldNodeLabels, + public Response removeFromClusterNodeLabels(Set oldNodeLabels, HttpServletRequest hsr) throws Exception { throw new NotImplementedException("Code is not implemented"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java index bb48476d652cc4..c221b86e984338 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java @@ -637,13 +637,13 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels, @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) @Override - public Response removeFromCluserNodeLabels( + public Response removeFromClusterNodeLabels( @QueryParam(RMWSConsts.LABELS) Set oldNodeLabels, @Context HttpServletRequest hsr) throws Exception { init(); RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr); return pipeline.getRootInterceptor() - .removeFromCluserNodeLabels(oldNodeLabels, hsr); + .removeFromClusterNodeLabels(oldNodeLabels, hsr); } @GET diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java index 05a088df781be7..e24ad47bf7c73d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java @@ -245,8 +245,8 @@ protected Response addToClusterNodeLabels(String user) throws Exception { null, createHttpServletRequest(user)); } - protected Response removeFromCluserNodeLabels(String user) throws Exception { - return routerWebService.removeFromCluserNodeLabels( + protected Response removeFromClusterNodeLabels(String user) throws Exception { + return routerWebService.removeFromClusterNodeLabels( null, createHttpServletRequest(user)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java index 67c9d671fb159c..68b8db6f334c19 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java @@ -215,7 +215,7 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels, } @Override - public Response removeFromCluserNodeLabels(Set oldNodeLabels, + public Response removeFromClusterNodeLabels(Set oldNodeLabels, HttpServletRequest hsr) throws Exception { return Response.status(Status.OK).build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java index 142a6511b93a64..31f8b8b990fd9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java @@ -248,9 +248,9 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels, } @Override - public Response removeFromCluserNodeLabels(Set oldNodeLabels, + public Response removeFromClusterNodeLabels(Set oldNodeLabels, HttpServletRequest hsr) throws Exception { - return getNextInterceptor().removeFromCluserNodeLabels(oldNodeLabels, hsr); + return getNextInterceptor().removeFromClusterNodeLabels(oldNodeLabels, hsr); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java index 14652435dac291..7491cbc2a9fc7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java @@ -132,7 +132,7 @@ public void testRouterWebServicesE2E() throws Exception { Response response4 = addToClusterNodeLabels(user); Assert.assertNotNull(response4); - Response response5 = removeFromCluserNodeLabels(user); + Response response5 = removeFromClusterNodeLabels(user); Assert.assertNotNull(response5); NodeLabelsInfo nodeLabelsInfo2 = getLabelsOnNode(user); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java index d3c619860a9cf2..868a953e9adfde 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java @@ -1203,10 +1203,10 @@ public void testAddToClusterNodeLabelsXML() throws Exception { /** * This test validates the correctness of - * {@link RMWebServiceProtocol#removeFromCluserNodeLabels()} inside Router. + * {@link RMWebServiceProtocol#removeFromClusterNodeLabels} inside Router. */ @Test(timeout = 2000) - public void testRemoveFromCluserNodeLabelsXML() + public void testRemoveFromClusterNodeLabelsXML() throws Exception { // Test with a wrong HTTP method From 1a1652256765e9a2c76036022426230ce181f2aa Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 20 Dec 2021 12:31:34 +0530 Subject: [PATCH 30/33] HADOOP-16908. Prune Jackson 1 from the codebase and restrict it's usage for future (#3789) Signed-off-by: Akira Ajisaka --- hadoop-common-project/hadoop-common/pom.xml | 18 +++++ .../hadoop/metrics2/MetricsJsonBuilder.java | 5 +- .../TestRouterClientRejectOverload.java | 3 +- .../fsdataset/impl/ProvidedVolumeImpl.java | 14 ++-- hadoop-project/pom.xml | 16 +++++ hadoop-tools/hadoop-azure/pom.xml | 12 ---- .../services/ListResultEntrySchema.java | 4 +- .../contracts/services/ListResultSchema.java | 4 +- .../azurebfs/oauth2/AzureADAuthenticator.java | 9 +-- .../azurebfs/services/AbfsHttpOperation.java | 10 +-- .../contract/ListResultSchemaTest.java | 2 +- .../tools/dynamometer/DynoInfraUtils.java | 11 +-- hadoop-tools/hadoop-resourceestimator/pom.xml | 18 +++++ .../sls/synthetic/SynthTraceJobProducer.java | 45 +++++++------ .../yarn/sls/TestSynthJobGeneration.java | 23 ++++--- .../pom.xml | 18 +++++ .../hadoop-yarn/hadoop-yarn-common/pom.xml | 18 +++++ .../yarn/util/DockerClientConfigHandler.java | 21 +++--- .../pom.xml | 18 +++++ .../hadoop-yarn-server-nodemanager/pom.xml | 18 +++++ .../NetworkTagMappingJsonManager.java | 8 ++- .../linux/runtime/RuncContainerRuntime.java | 9 +-- .../runc/ImageTagToManifestPlugin.java | 3 +- .../runc/RuncContainerExecutorConfig.java | 67 ++++++++++--------- .../runtime/TestDockerContainerRuntime.java | 1 - .../runtime/TestImageTagToManifestPlugin.java | 3 +- .../runtime/TestRuncContainerRuntime.java | 7 +- .../pom.xml | 18 +++++ .../resource/ResourceProfilesManagerImpl.java | 3 +- .../documentstore/DocumentStoreTestUtils.java | 3 +- .../documentstore/JsonUtils.java | 13 ++-- pom.xml | 8 +++ 32 files changed, 290 insertions(+), 140 deletions(-) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index a75ab5ecc4569f..84da42a3204fc1 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -147,6 +147,24 @@ com.sun.jersey jersey-json compile + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + com.sun.jersey diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java index 1d62c0a29fca4d..3a9be12803143f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java @@ -21,8 +21,9 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.ObjectWriter; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java index cc7f5a61a22ff1..71ec747af4c304 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java @@ -50,7 +50,8 @@ import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.test.GenericTestUtils; -import org.codehaus.jackson.map.ObjectMapper; + +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.After; import org.junit.Rule; import org.junit.Test; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java index 7c6562f58d16ee..eae119712f7c4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java @@ -32,6 +32,10 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -60,10 +64,6 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Timer; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.ObjectReader; -import org.codehaus.jackson.map.ObjectWriter; import org.apache.hadoop.classification.VisibleForTesting; @@ -371,14 +371,11 @@ public void releaseReservedSpace(long bytesToRelease) { private static final ObjectWriter WRITER = new ObjectMapper().writerWithDefaultPrettyPrinter(); - private static final ObjectReader READER = - new ObjectMapper().reader(ProvidedBlockIteratorState.class); private static class ProvidedBlockIteratorState { ProvidedBlockIteratorState() { iterStartMs = Time.now(); lastSavedMs = iterStartMs; - atEnd = false; lastBlockId = -1L; } @@ -390,9 +387,6 @@ private static class ProvidedBlockIteratorState { @JsonProperty private long iterStartMs; - @JsonProperty - private boolean atEnd; - // The id of the last block read when the state of the iterator is saved. // This implementation assumes that provided blocks are returned // in sorted order of the block ids. diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index cc45975a3f0771..681143c336edff 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -870,6 +870,22 @@ stax stax-api + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 1896e15d27018c..a9f58e39c6a4bf 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -178,18 +178,6 @@ compile - - org.codehaus.jackson - jackson-mapper-asl - compile - - - - org.codehaus.jackson - jackson-core-asl - compile - - org.wildfly.openssl wildfly-openssl diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultEntrySchema.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultEntrySchema.java index cdf3decdc98bce..a9883dd2ce5fc4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultEntrySchema.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultEntrySchema.java @@ -18,8 +18,8 @@ package org.apache.hadoop.fs.azurebfs.contracts.services; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultSchema.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultSchema.java index e3519fb429bff1..dc7da04b5bd4f5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultSchema.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultSchema.java @@ -20,8 +20,8 @@ import java.util.List; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java index aad805fe999199..dd4ec7c2009c2b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java @@ -30,9 +30,10 @@ import java.util.Map; import org.apache.hadoop.util.Preconditions; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.JsonToken; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -451,7 +452,7 @@ private static AzureADToken parseTokenFromStream( long expiresOnInSecs = -1; JsonFactory jf = new JsonFactory(); - JsonParser jp = jf.createJsonParser(httpResponseStream); + JsonParser jp = jf.createParser(httpResponseStream); String fieldName, fieldValue; jp.nextToken(); while (jp.hasCurrentToken()) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 6e13d4cec67378..413bf3686898ba 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -30,11 +30,11 @@ import org.apache.hadoop.fs.azurebfs.utils.UriUtils; import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.JsonToken; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -467,7 +467,7 @@ private void processStorageErrorResponse() { return; } JsonFactory jf = new JsonFactory(); - try (JsonParser jp = jf.createJsonParser(stream)) { + try (JsonParser jp = jf.createParser(stream)) { String fieldName, fieldValue; jp.nextToken(); // START_OBJECT - { jp.nextToken(); // FIELD_NAME - "error": diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ListResultSchemaTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ListResultSchemaTest.java index 8a33ea5de06413..3f6a4872c5d171 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ListResultSchemaTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ListResultSchemaTest.java @@ -20,7 +20,7 @@ import java.io.IOException; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; import org.apache.hadoop.fs.azurebfs.contracts.services.ListResultEntrySchema; diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/DynoInfraUtils.java b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/DynoInfraUtils.java index ee0810d6439d7c..f6c8a6ac4d58be 100644 --- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/DynoInfraUtils.java +++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/DynoInfraUtils.java @@ -54,9 +54,10 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.JsonToken; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; import org.slf4j.Logger; @@ -484,7 +485,7 @@ static Set parseStaleDataNodeList(String liveNodeJsonString, final Set dataNodesToReport = new HashSet<>(); JsonFactory fac = new JsonFactory(); - JsonParser parser = fac.createJsonParser(IOUtils + JsonParser parser = fac.createParser(IOUtils .toInputStream(liveNodeJsonString, StandardCharsets.UTF_8.name())); int objectDepth = 0; @@ -554,7 +555,7 @@ static String fetchNameNodeJMXValue(Properties nameNodeProperties, } InputStream in = conn.getInputStream(); JsonFactory fac = new JsonFactory(); - JsonParser parser = fac.createJsonParser(in); + JsonParser parser = fac.createParser(in); if (parser.nextToken() != JsonToken.START_OBJECT || parser.nextToken() != JsonToken.FIELD_NAME || !parser.getCurrentName().equals("beans") diff --git a/hadoop-tools/hadoop-resourceestimator/pom.xml b/hadoop-tools/hadoop-resourceestimator/pom.xml index 961aa2933e3b02..1ef264ba2120b2 100644 --- a/hadoop-tools/hadoop-resourceestimator/pom.xml +++ b/hadoop-tools/hadoop-resourceestimator/pom.xml @@ -81,6 +81,24 @@ com.sun.jersey jersey-json + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + junit diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java index 7a3e22bd4cb5b0..3527d6b7668d88 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java @@ -17,6 +17,13 @@ */ package org.apache.hadoop.yarn.sls.synthetic; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonFactoryBuilder; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.math3.distribution.AbstractRealDistribution; @@ -30,18 +37,13 @@ import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.sls.appmaster.MRAMSimulator; -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.JsonMappingException; -import org.codehaus.jackson.map.ObjectMapper; import javax.xml.bind.annotation.XmlRootElement; import java.io.IOException; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; -import static org.codehaus.jackson.JsonParser.Feature.INTERN_FIELD_NAMES; -import static org.codehaus.jackson.map.DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES; +import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; /** * This is a JobStoryProducer that operates from distribution of different @@ -84,15 +86,16 @@ public SynthTraceJobProducer(Configuration conf, Path path) this.conf = conf; this.rand = new JDKRandomGenerator(); - ObjectMapper mapper = new ObjectMapper(); - mapper.configure(INTERN_FIELD_NAMES, true); + JsonFactoryBuilder jsonFactoryBuilder = new JsonFactoryBuilder(); + jsonFactoryBuilder.configure(JsonFactory.Feature.INTERN_FIELD_NAMES, true); + ObjectMapper mapper = new ObjectMapper(jsonFactoryBuilder.build()); mapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, false); FileSystem ifs = path.getFileSystem(conf); FSDataInputStream fileIn = ifs.open(path); // Initialize the random generator and the seed - this.trace = mapper.readValue(fileIn, Trace.class); + this.trace = mapper.readValue(fileIn.getWrappedStream(), Trace.class); this.seed = trace.rand_seed; this.rand.setSeed(seed); // Initialize the trace @@ -538,9 +541,9 @@ public Sample(@JsonProperty("val") Double val, if(val!=null){ if(std==null){ // Constant - if(dist!=null || discrete!=null || weights!=null){ - throw new JsonMappingException("Instantiation of " + Sample.class - + " failed"); + if (dist != null || discrete != null || weights != null) { + throw JsonMappingException + .from((JsonParser) null, "Instantiation of " + Sample.class + " failed"); } mode = Mode.CONST; this.val = val; @@ -550,9 +553,9 @@ public Sample(@JsonProperty("val") Double val, this.weights = null; } else { // Distribution - if(discrete!=null || weights != null){ - throw new JsonMappingException("Instantiation of " + Sample.class - + " failed"); + if (discrete != null || weights != null) { + throw JsonMappingException + .from((JsonParser) null, "Instantiation of " + Sample.class + " failed"); } mode = Mode.DIST; this.val = val; @@ -563,9 +566,9 @@ public Sample(@JsonProperty("val") Double val, } } else { // Discrete - if(discrete==null){ - throw new JsonMappingException("Instantiation of " + Sample.class - + " failed"); + if (discrete == null) { + throw JsonMappingException + .from((JsonParser) null, "Instantiation of " + Sample.class + " failed"); } mode = Mode.DISC; this.val = 0; @@ -576,9 +579,9 @@ public Sample(@JsonProperty("val") Double val, weights = new ArrayList<>(Collections.nCopies( discrete.size(), 1.0)); } - if(weights.size() != discrete.size()){ - throw new JsonMappingException("Instantiation of " + Sample.class - + " failed"); + if (weights.size() != discrete.size()) { + throw JsonMappingException + .from((JsonParser) null, "Instantiation of " + Sample.class + " failed"); } this.weights = weights; } diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSynthJobGeneration.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSynthJobGeneration.java index 0792eece51e8cc..14e74751577cf4 100644 --- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSynthJobGeneration.java +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSynthJobGeneration.java @@ -19,11 +19,14 @@ import org.apache.commons.math3.random.JDKRandomGenerator; import org.apache.hadoop.yarn.api.records.ExecutionType; -import org.codehaus.jackson.map.JsonMappingException; -import org.codehaus.jackson.map.ObjectMapper; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.sls.synthetic.SynthJob; import org.apache.hadoop.yarn.sls.synthetic.SynthTraceJobProducer; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonFactoryBuilder; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -32,12 +35,10 @@ import java.io.IOException; import java.util.Arrays; +import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.codehaus.jackson.JsonParser.Feature.INTERN_FIELD_NAMES; -import static org.codehaus.jackson.map.DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES; - /** * Simple test class driving the {@code SynthTraceJobProducer}, and validating * jobs produce are within expected range. @@ -56,8 +57,9 @@ public void testWorkloadGenerateTime() + "{\"time\": 60, \"weight\": 2}," + "{\"time\": 90, \"weight\": 1}" + "]}"; - ObjectMapper mapper = new ObjectMapper(); - mapper.configure(INTERN_FIELD_NAMES, true); + JsonFactoryBuilder jsonFactoryBuilder = new JsonFactoryBuilder(); + jsonFactoryBuilder.configure(JsonFactory.Feature.INTERN_FIELD_NAMES, true); + ObjectMapper mapper = new ObjectMapper(jsonFactoryBuilder.build()); mapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, false); SynthTraceJobProducer.Workload wl = mapper.readValue(workloadJson, SynthTraceJobProducer.Workload.class); @@ -176,8 +178,9 @@ public void testStream() throws IllegalArgumentException, IOException { @Test public void testSample() throws IOException { - ObjectMapper mapper = new ObjectMapper(); - mapper.configure(INTERN_FIELD_NAMES, true); + JsonFactoryBuilder jsonFactoryBuilder = new JsonFactoryBuilder(); + jsonFactoryBuilder.configure(JsonFactory.Feature.INTERN_FIELD_NAMES, true); + ObjectMapper mapper = new ObjectMapper(jsonFactoryBuilder.build()); mapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, false); JDKRandomGenerator rand = new JDKRandomGenerator(); @@ -235,7 +238,7 @@ public void testSample() throws IOException { mapper.readValue(invalidDistJson, SynthTraceJobProducer.Sample.class); Assert.fail(); } catch (JsonMappingException e) { - Assert.assertTrue(e.getMessage().startsWith("Instantiation of")); + Assert.assertTrue(e.getMessage().startsWith("Cannot construct instance of")); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml index 57edd96871e518..cbcd13abb955c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml @@ -95,6 +95,24 @@ com.sun.jersey jersey-json ${jersey.version} + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index 489236acf398bb..e2146e7346968b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -158,6 +158,24 @@ com.sun.jersey jersey-json + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + com.sun.jersey.contribs diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java index c996225c9a75e0..91002e40d6a04c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java @@ -28,11 +28,12 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.security.DockerCredentialTokenIdentifier; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.node.ObjectNode; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.slf4j.LoggerFactory; import java.io.File; @@ -97,13 +98,13 @@ public static Credentials readCredentialsFromConfigFile(Path configFile, // Parse the JSON and create the Tokens/Credentials. ObjectMapper mapper = new ObjectMapper(); - JsonFactory factory = mapper.getJsonFactory(); - JsonParser parser = factory.createJsonParser(contents); + JsonFactory factory = mapper.getFactory(); + JsonParser parser = factory.createParser(contents); JsonNode rootNode = mapper.readTree(parser); Credentials credentials = new Credentials(); if (rootNode.has(CONFIG_AUTHS_KEY)) { - Iterator iter = rootNode.get(CONFIG_AUTHS_KEY).getFieldNames(); + Iterator iter = rootNode.get(CONFIG_AUTHS_KEY).fieldNames(); for (; iter.hasNext();) { String registryUrl = iter.next(); String registryCred = rootNode.get(CONFIG_AUTHS_KEY) @@ -169,14 +170,14 @@ public static boolean writeDockerCredentialsToPath(File outConfigFile, DockerCredentialTokenIdentifier ti = (DockerCredentialTokenIdentifier) tk.decodeIdentifier(); ObjectNode registryCredNode = mapper.createObjectNode(); - registryUrlNode.put(ti.getRegistryUrl(), registryCredNode); + registryUrlNode.set(ti.getRegistryUrl(), registryCredNode); registryCredNode.put(CONFIG_AUTH_KEY, new String(tk.getPassword(), Charset.forName("UTF-8"))); LOG.debug("Prepared token for write: {}", tk); } } if (foundDockerCred) { - rootNode.put(CONFIG_AUTHS_KEY, registryUrlNode); + rootNode.set(CONFIG_AUTHS_KEY, registryUrlNode); String json = mapper.writerWithDefaultPrettyPrinter() .writeValueAsString(rootNode); FileUtils.writeStringToFile( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index 06e98fd2b7c825..9c87f44a6d73d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -94,6 +94,24 @@ com.sun.jersey jersey-json + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + com.sun.jersey.contribs diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 79bb6cfa22de46..b51d51cffde957 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -154,6 +154,24 @@ com.sun.jersey jersey-json + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + com.sun.jersey.contribs diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkTagMappingJsonManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkTagMappingJsonManager.java index 36cb5e3b08ca0c..cc2ded4422b71b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkTagMappingJsonManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkTagMappingJsonManager.java @@ -25,15 +25,17 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; -import org.codehaus.jackson.annotate.JsonIgnore; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.ObjectMapper; /** * The NetworkTagMapping JsonManager implementation. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncContainerRuntime.java index 78ed5a6b97767f..e43f7788d78b16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncContainerRuntime.java @@ -62,8 +62,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; import org.apache.hadoop.yarn.server.nodemanager.containermanager.volume.csi.ContainerVolumePublisher; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerExecContext; -import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.map.ObjectMapper; import java.io.File; import java.io.IOException; @@ -81,6 +79,9 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_RUNC_IMAGE_TAG_TO_MANIFEST_PLUGIN; import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_RUNC_LAYER_MOUNTS_TO_KEEP; import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_REAP_RUNC_LAYER_MOUNTS_INTERVAL; @@ -642,7 +643,7 @@ protected List extractImageEnv(File config) throws IOException { if (envNode.isMissingNode()) { return null; } - return mapper.readValue(envNode, List.class); + return mapper.readValue(envNode.traverse(), List.class); } @SuppressWarnings("unchecked") @@ -653,7 +654,7 @@ protected List extractImageEntrypoint(File config) if (entrypointNode.isMissingNode()) { return null; } - return mapper.readValue(entrypointNode, List.class); + return mapper.readValue(entrypointNode.traverse(), List.class); } private RuncContainerExecutorConfig createRuncContainerExecutorConfig( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/ImageTagToManifestPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/ImageTagToManifestPlugin.java index 629785dfcd7f14..fbec3ee6f5ea68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/ImageTagToManifestPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/ImageTagToManifestPlugin.java @@ -29,7 +29,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.concurrent.HadoopExecutors; -import org.codehaus.jackson.map.ObjectMapper; import java.io.BufferedReader; import java.io.File; @@ -45,6 +44,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import com.fasterxml.jackson.databind.ObjectMapper; + import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_RUNC_CACHE_REFRESH_INTERVAL; import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_RUNC_IMAGE_TOPLEVEL_DIR; import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NUM_MANIFESTS_TO_CACHE; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/RuncContainerExecutorConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/RuncContainerExecutorConfig.java index 88a01a22e13edf..3333b820c893c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/RuncContainerExecutorConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/RuncContainerExecutorConfig.java @@ -20,12 +20,13 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc; import org.apache.hadoop.classification.InterfaceStability; -import org.codehaus.jackson.annotate.JsonRawValue; -import org.codehaus.jackson.map.annotate.JsonSerialize; import java.util.List; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonRawValue; + /** * This class is used by the * {@link org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.RuncContainerRuntime} @@ -35,7 +36,7 @@ * a JSON object named ociRuntimeConfig that mirrors the * OCI runtime specification. */ -@JsonSerialize(include=JsonSerialize.Inclusion.NON_DEFAULT) +@JsonInclude(JsonInclude.Include.NON_DEFAULT) @InterfaceStability.Unstable public class RuncContainerExecutorConfig { final private String version; @@ -164,7 +165,7 @@ public RuncContainerExecutorConfig(String version, String runAsUser, /** * This class is a Java representation of an OCI image layer. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) @InterfaceStability.Unstable public static class OCILayer { final private String mediaType; @@ -192,7 +193,7 @@ public OCILayer() { * This class is a Java representation of the OCI Runtime Specification. */ @InterfaceStability.Unstable - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class OCIRuntimeConfig { final private OCIRootConfig root; final private List mounts; @@ -254,7 +255,7 @@ public OCIRuntimeConfig(OCIRootConfig root, List mounts, * This class is a Java representation of the oci root config section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class OCIRootConfig { public String getPath() { return path; @@ -281,7 +282,7 @@ public OCIRootConfig() { * This class is a Java representation of the oci mount section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class OCIMount { final private String destination; final private String type; @@ -329,7 +330,7 @@ public OCIMount() { * This class is a Java representation of the oci process section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class OCIProcessConfig { final private boolean terminal; final private ConsoleSize consoleSize; @@ -422,7 +423,7 @@ public OCIProcessConfig() { * This class is a Java representation of the console size section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class ConsoleSize { public int getHeight() { return height; @@ -450,7 +451,7 @@ public ConsoleSize() { * This class is a Java representation of the rlimits section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class RLimits { public String getType() { return type; @@ -484,7 +485,7 @@ public RLimits() { * This class is a Java representation of the capabilities section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class Capabilities { final private List effective; final private List bounding; @@ -554,7 +555,7 @@ public User() { * This class is a Java representation of the oci hooks section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class OCIHooksConfig { final private List prestart; final private List poststart; @@ -587,7 +588,7 @@ public OCIHooksConfig() { * This class is a Java representation of the hook type section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class HookType { final private String path; final private List args; @@ -650,7 +651,7 @@ public OCIAnnotationsConfig() { * This class is a Java representation of the oci linux config section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class OCILinuxConfig { final private List namespaces; final private List uidMappings; @@ -768,7 +769,7 @@ public Namespace() { * This class is a Java representation of the idmapping section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class IDMapping { final private int containerID; final private int hostID; @@ -802,7 +803,7 @@ public IDMapping() { * This class is a Java representation of the device section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class Device { final private String type; final private String path; @@ -861,7 +862,7 @@ public Device() { * This class is a Java representation of the resources section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class Resources { final private List device; final private Memory memory; @@ -927,7 +928,7 @@ public Resources() { * This class is a Java representation of the device section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class Device { final private boolean allow; final private String type; @@ -973,7 +974,7 @@ public Device() { * This class is a Java representation of the memory section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class Memory { final private long limit; final private long reservation; @@ -1032,7 +1033,7 @@ public Memory() { * This class is a Java representation of the cpu section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class CPU { final private long quota; final private long period; @@ -1092,7 +1093,7 @@ public CPU() { * This class is a Java representation of the blockio section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class BlockIO { final private int weight; final private int leafWeight; @@ -1153,7 +1154,7 @@ public BlockIO() { * This class is a Java representation of the weight device section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class WeightDevice { final private long major; final private long minor; @@ -1193,7 +1194,7 @@ public WeightDevice() { * This class is a Java representation of the throttle device section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class ThrottleDevice { final private long major; final private long minor; @@ -1227,7 +1228,7 @@ public ThrottleDevice() { * This class is a Java representation of the huge page limits section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class HugePageLimits { final private String pageSize; final private long limit; @@ -1254,7 +1255,7 @@ public HugePageLimits() { * This class is a Java representation of the network section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class Network { final private int classID; final private List priorities; @@ -1280,7 +1281,7 @@ public Network() { * This class is a Java representation of the network priority section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class NetworkPriority { final private String name; final private int priority; @@ -1308,7 +1309,7 @@ public NetworkPriority() { * This class is a Java representation of the pid section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class PID { final private long limit; @@ -1329,7 +1330,7 @@ public PID() { * This class is a Java representation of the rdma section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class RDMA { final private int hcaHandles; final private int hcaObjects; @@ -1357,7 +1358,7 @@ public RDMA() { * This class is a Java representation of the intelrdt section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class IntelRdt { final private String closID; final private String l3CacheSchema; @@ -1391,7 +1392,7 @@ public IntelRdt() { * This class is a Java representation of the sysctl section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class Sysctl { // for kernel params } @@ -1400,7 +1401,7 @@ public static class Sysctl { * This class is a Java representation of the seccomp section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class Seccomp { final private String defaultAction; final private List architectures; @@ -1433,7 +1434,7 @@ public Seccomp() { * This class is a Java representation of the syscall section * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class Syscall { final private List names; final private String action; @@ -1466,7 +1467,7 @@ public Syscall() { * This class is a Java representation of the seccomp arguments * of the OCI Runtime Specification. */ - @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT) + @JsonInclude(JsonInclude.Include.NON_DEFAULT) public static class SeccompArg { final private int index; final private long value; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index bb1abf51df03f2..af1f7571264e47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; -import org.codehaus.jackson.map.ObjectMapper; import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestImageTagToManifestPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestImageTagToManifestPlugin.java index 73bfa026f145c2..3c2a951597ec91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestImageTagToManifestPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestImageTagToManifestPlugin.java @@ -24,7 +24,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageManifest; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin; -import org.codehaus.jackson.map.ObjectMapper; + +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestRuncContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestRuncContainerRuntime.java index 1e06d03dda4b27..8a541bbe1ae32d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestRuncContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestRuncContainerRuntime.java @@ -58,8 +58,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; -import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.map.ObjectMapper; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -501,7 +502,7 @@ private RuncContainerExecutorConfig verifyRuncConfig(File configFile) JsonNode configNode = mapper.readTree(configFile); RuncContainerExecutorConfig runcContainerExecutorConfig = - mapper.readValue(configNode, RuncContainerExecutorConfig.class); + mapper.readValue(configNode.traverse(), RuncContainerExecutorConfig.class); configSize = configNode.size(); OCIRuntimeConfig ociRuntimeConfig = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index f17ef707ef4b41..1a0f4c00f77b35 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -109,6 +109,24 @@ com.sun.jersey jersey-json + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + com.sun.jersey.contribs diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java index 1f101ee3860a34..24cb34327b7457 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.resource; import org.apache.hadoop.classification.VisibleForTesting; + +import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -29,7 +31,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; -import org.codehaus.jackson.map.ObjectMapper; import java.io.File; import java.io.IOException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTestUtils.java index cf57085023f267..5d442152fe9ba8 100755 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTestUtils.java @@ -23,11 +23,12 @@ import org.apache.hadoop.yarn.server.timelineservice.documentstore.collection.document.entity.TimelineEntityDocument; import org.apache.hadoop.yarn.server.timelineservice.documentstore.collection.document.flowactivity.FlowActivityDocument; import org.apache.hadoop.yarn.server.timelineservice.documentstore.collection.document.flowrun.FlowRunDocument; -import org.codehaus.jackson.type.TypeReference; import java.io.IOException; import java.util.List; +import com.fasterxml.jackson.core.type.TypeReference; + /** * This is util class for baking sample TimelineEntities data for test. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/JsonUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/JsonUtils.java index c1da4f6ce43602..a644bc1d191036 100755 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/JsonUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/JsonUtils.java @@ -18,12 +18,12 @@ package org.apache.hadoop.yarn.server.timelineservice.documentstore; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; - import java.io.IOException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + /** * A simple util class for Json SerDe. */ @@ -34,8 +34,7 @@ private JsonUtils(){} private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); static { - OBJECT_MAPPER.configure( - DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); + OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); } /** @@ -48,7 +47,7 @@ private JsonUtils(){} * @throws IOException if Json String is not valid or error * while deserialization */ - public static T fromJson(final String jsonStr, final TypeReference type) + public static T fromJson(final String jsonStr, final TypeReference type) throws IOException { return OBJECT_MAPPER.readValue(jsonStr, type); } diff --git a/pom.xml b/pom.xml index 09f1ccb6d24741..6e295b838333dd 100644 --- a/pom.xml +++ b/pom.xml @@ -273,6 +273,14 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.** + + true + Use Fasterxml Jackson 2 dependency in place of org.codehaus Jackson 1 + + org.codehaus.jackson.** + static org.codehaus.jackson.** + + From ebd3bcbcf721e62fcaa2c4690d04aa9fb6534226 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 20 Dec 2021 15:32:57 +0530 Subject: [PATCH 31/33] YARN-11047. ResourceManager and NodeManager unable to connect to Hbase when ATSv2 is enabled (#3802) --- hadoop-project/pom.xml | 2 ++ hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 +-- hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 4 +-- .../pom.xml | 7 +++++ .../pom.xml | 26 +++++++++++++++++++ .../pom.xml | 22 ++++++++++++++++ .../pom.xml | 16 ++++++++++++ .../pom.xml | 16 ++++++++++++ hadoop-yarn-project/pom.xml | 5 ++-- 9 files changed, 95 insertions(+), 7 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 681143c336edff..c7f86abfa42664 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -2418,6 +2418,7 @@ ${hbase.one.version} 2.8.5 12.0.1 + 4.0 hadoop-yarn-server-timelineservice-hbase-server-1 @@ -2446,6 +2447,7 @@ 2.8.5 11.0.2 hadoop-yarn-server-timelineservice-hbase-server-2 + 4.0 9.3.27.v20190418 diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 5eccaadeb605b5..f305c2744efdf9 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -124,7 +124,7 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" nodemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*" - hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" before HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.nodemanager.NodeManager' # Backwards compatibility if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then @@ -151,7 +151,7 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" resourcemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*" - hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" before HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' # Backwards compatibility if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 4508ad38b66f0c..a4340d08adbb84 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -220,7 +220,7 @@ goto :eof :resourcemanager set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\* - set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\* + set CLASSPATH=%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\*;%CLASSPATH% set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager set YARN_OPTS=%YARN_OPTS% %YARN_RESOURCEMANAGER_OPTS% if defined YARN_RESOURCEMANAGER_HEAPSIZE ( @@ -268,7 +268,7 @@ goto :eof :nodemanager set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\* - set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\* + set CLASSPATH=HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\*;%CLASSPATH% set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager set YARN_OPTS=%YARN_OPTS% -server %HADOOP_NODEMANAGER_OPTS% if defined YARN_NODEMANAGER_HEAPSIZE ( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml index fee962575e80c6..b2f2d3a966c4aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml @@ -181,6 +181,13 @@ test + + com.google.inject + guice + ${hbase-compatible-guice.version} + test + + com.sun.jersey jersey-client diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml index f6ea8660e83112..0bedcfcb27fa47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml @@ -42,6 +42,10 @@ com.google.guava guava + + com.google.inject + guice + @@ -66,6 +70,18 @@ ${hbase-compatible-guava.version} + + com.google.inject + guice + ${hbase-compatible-guice.version} + + + com.google.guava + guava + + + + org.apache.hadoop hadoop-annotations @@ -121,6 +137,12 @@ org.apache.hadoop hadoop-yarn-common provided + + + com.google.inject + guice + + @@ -137,6 +159,10 @@ com.google.guava guava + + com.google.inject + guice + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml index 247981d61d6722..11be84e3c11fa0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml @@ -56,6 +56,10 @@ com.google.guava guava + + com.google.inject + guice + @@ -64,6 +68,12 @@ org.apache.hadoop hadoop-yarn-server-applicationhistoryservice provided + + + com.google.inject + guice + + From b4c7e1b6a4f9789ab468dd37f0b8df47a60d5a3c Mon Sep 17 00:00:00 2001 From: secfree Date: Mon, 20 Dec 2021 18:16:22 +0800 Subject: [PATCH 32/33] HDFS-16168. Fix TestHDFSFileSystemContract.testAppend timeout (#3815) --- .../org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java index 3a8528968dc6ce..9de5813872f1b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java @@ -63,7 +63,12 @@ protected String getDefaultWorkingDirectory() { return defaultWorkingDirectory; } - @Test(timeout = 60000) + @Override + protected int getGlobalTimeout() { + return 60 * 1000; + } + + @Test public void testAppend() throws IOException { AppendTestUtil.testAppend(fs, new Path("/testAppend/f")); } From 6a0de4fcb79ae93f3fb70941aa46f982cc05e5e9 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Mon, 20 Dec 2021 19:28:55 +0800 Subject: [PATCH 33/33] HDFS-16386. Reduce DataNode load when FsDatasetAsyncDiskService is working. (#3806) --- .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +++ .../fsdataset/impl/FsDatasetAsyncDiskService.java | 12 ++++++++++-- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 10 ++++++++++ .../metrics/TestSystemMetricsPublisher.java | 11 +++++++---- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 0526f1e4412dc7..2e68cb6a1b511f 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -143,6 +143,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final long DFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT = 0; public static final String DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY = "dfs.datanode.fsdatasetcache.max.threads.per.volume"; public static final int DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_DEFAULT = 4; + public static final String DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY = + "dfs.datanode.fsdatasetasyncdisk.max.threads.per.volume"; + public static final int DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT = 4; public static final String DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC = "dfs.datanode.lazywriter.interval.sec"; public static final int DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC = 60; public static final String DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_KEY = "dfs.datanode.ram.disk.replica.tracker"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java index 7d5f33b8b8cf83..db4987e25ac98e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java @@ -30,6 +30,8 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -65,7 +67,7 @@ class FsDatasetAsyncDiskService { // ThreadPool core pool size private static final int CORE_THREADS_PER_VOLUME = 1; // ThreadPool maximum pool size - private static final int MAXIMUM_THREADS_PER_VOLUME = 4; + private final int maxNumThreadsPerVolume; // ThreadPool keep-alive time for threads over core pool size private static final long THREADS_KEEP_ALIVE_SECONDS = 60; @@ -90,6 +92,12 @@ class FsDatasetAsyncDiskService { this.datanode = datanode; this.fsdatasetImpl = fsdatasetImpl; this.threadGroup = new ThreadGroup(getClass().getSimpleName()); + maxNumThreadsPerVolume = datanode.getConf().getInt( + DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY, + DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT); + Preconditions.checkArgument(maxNumThreadsPerVolume > 0, + DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY + + " must be a positive integer."); } private void addExecutorForVolume(final FsVolumeImpl volume) { @@ -110,7 +118,7 @@ public Thread newThread(Runnable r) { }; ThreadPoolExecutor executor = new ThreadPoolExecutor( - CORE_THREADS_PER_VOLUME, MAXIMUM_THREADS_PER_VOLUME, + CORE_THREADS_PER_VOLUME, maxNumThreadsPerVolume, THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS, new LinkedBlockingQueue(), threadFactory); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 7bcbccd81728ce..9422c1d6c3c58e 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2982,6 +2982,16 @@ + + dfs.datanode.fsdatasetasyncdisk.max.threads.per.volume + 4 + + The maximum number of threads per volume used to process async disk + operations on the datanode. These threads consume I/O and CPU at the + same time. This will affect normal data node operations. + + + dfs.cachereport.intervalMsec 10000 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 7bea24c8416c95..146a931e5acef3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -87,11 +87,14 @@ public static Collection data() { private static TimelineServiceV1Publisher metricsPublisher; private static TimelineStore store; - @Parameterized.Parameter - public boolean rmTimelineServerV1PublisherBatchEnabled; + private boolean rmTimelineServerV1PublisherBatchEnabled; + private int rmTimelineServerV1PublisherInterval; - @Parameterized.Parameter(1) - public int rmTimelineServerV1PublisherInterval; + public TestSystemMetricsPublisher(boolean rmTimelineServerV1PublisherBatchEnabled, + int rmTimelineServerV1PublisherInterval) { + this.rmTimelineServerV1PublisherBatchEnabled = rmTimelineServerV1PublisherBatchEnabled; + this.rmTimelineServerV1PublisherInterval = rmTimelineServerV1PublisherInterval; + } @Before public void setup() throws Exception {