Skip to content

YARN-10869. CS considers only the default maximum-allocation-mb/vcore property as a maximum when it creates dynamic queues #3225

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
szilard-nemeth merged 1 commit into from
Jul 29, 2021
Merged

YARN-10869. CS considers only the default maximum-allocation-mb/vcore property as a maximum when it creates dynamic queues #3225

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion ...ava/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -518,8 +518,14 @@ protected void setDynamicQueueProperties(

private void setupMaximumAllocation(CapacitySchedulerConfiguration csConf) {
String myQueuePath = getQueuePath();
/* YARN-10869: When using AutoCreatedLeafQueues, the passed configuration
* object is a cloned one containing only the template configs
* (see ManagedParentQueue#getLeafQueueConfigs). To ensure that the actual
* cluster maximum allocation is fetched the original config object should
* be used.
*/
Resource clusterMax = ResourceUtils
.fetchMaximumAllocationFromConfig(csConf);
.fetchMaximumAllocationFromConfig(this.csContext.getConfiguration());
Resource queueMax = csConf.getQueueMaximumAllocation(myQueuePath);

maximumAllocation = Resources.clone(
Expand Down
14 changes: 14 additions & 0 deletions ...hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1043,6 +1043,11 @@ public Resource getQueueMaximumAllocation(String queue) {
}
}

public void setQueueMaximumAllocation(String queue, String maximumAllocation) {
String queuePrefix = getQueuePrefix(queue);
set(queuePrefix + MAXIMUM_ALLOCATION, maximumAllocation);
}

public long getQueueMaximumAllocationMb(String queue) {
String queuePrefix = getQueuePrefix(queue);
return getInt(queuePrefix + MAXIMUM_ALLOCATION_MB, (int)UNDEFINED);
Expand Down Expand Up @@ -2465,6 +2470,15 @@ public void setAutoCreatedLeafQueueConfigDefaultNodeLabelExpression(String
setDefaultNodeLabelExpression(leafQueueConfPrefix, expression);
}

@Private
@VisibleForTesting
public void setAutoCreatedLeafQueueConfigMaximumAllocation(String
queuePath, String expression) {
String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
queuePath);
setQueueMaximumAllocation(leafQueueConfPrefix, expression);
}

public static String getUnits(String resourceValue) {
String units;
for (int i = 0; i < resourceValue.length(); i++) {
Expand Down
32 changes: 29 additions & 3 deletions .../server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,9 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration(
conf.setAutoCreatedLeafQueueConfigMaxCapacity(C, 100.0f);
conf.setAutoCreatedLeafQueueConfigUserLimit(C, 100);
conf.setAutoCreatedLeafQueueConfigUserLimitFactor(C, 3.0f);
conf.setAutoCreatedLeafQueueConfigUserLimitFactor(C, 3.0f);
conf.setAutoCreatedLeafQueueConfigMaximumAllocation(C,
"memory-mb=10240,vcores=6");

conf.setAutoCreatedLeafQueueTemplateCapacityByLabel(C, NODEL_LABEL_GPU,
NODE_LABEL_GPU_TEMPLATE_CAPACITY);
Expand Down Expand Up @@ -563,9 +566,31 @@ protected CapacitySchedulerConfiguration setupSchedulerConfiguration() {
schedConf.setInt(YarnConfiguration.RESOURCE_TYPES
+ ".memory-mb.maximum-allocation", 16384);


return new CapacitySchedulerConfiguration(schedConf);
}

protected void setSchedulerMinMaxAllocation(CapacitySchedulerConfiguration conf) {
unsetMinMaxAllocation(conf);

conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 1);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 8);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1024);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 18384);

}

private void unsetMinMaxAllocation(CapacitySchedulerConfiguration conf) {
conf.unset(YarnConfiguration.RESOURCE_TYPES
+ ".vcores.minimum-allocation");
conf.unset(YarnConfiguration.RESOURCE_TYPES
+ ".vcores.maximum-allocation");
conf.unset(YarnConfiguration.RESOURCE_TYPES
+ ".memory-mb.minimum-allocation");
conf.unset(YarnConfiguration.RESOURCE_TYPES
+ ".memory-mb.maximum-allocation");
}

protected MockRM setupSchedulerInstance() throws Exception {

if (mockRM != null) {
Expand Down Expand Up @@ -663,10 +688,11 @@ protected void validateUserAndAppLimits(
}

protected void validateContainerLimits(
AutoCreatedLeafQueue autoCreatedLeafQueue) {
assertEquals(8,
AutoCreatedLeafQueue autoCreatedLeafQueue, int vCoreLimit,
long memorySize) {
assertEquals(vCoreLimit,
autoCreatedLeafQueue.getMaximumAllocation().getVirtualCores());
assertEquals(16384,
assertEquals(memorySize,
autoCreatedLeafQueue.getMaximumAllocation().getMemorySize());
}

Expand Down
33 changes: 31 additions & 2 deletions ...arn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ public void testAutoCreateLeafQueueCreation() throws Exception {
expectedChildQueueAbsCapacity, accessibleNodeLabelsOnC);

validateUserAndAppLimits(autoCreatedLeafQueue, 4000, 4000);
validateContainerLimits(autoCreatedLeafQueue);
validateContainerLimits(autoCreatedLeafQueue, 6, 10240);

assertTrue(autoCreatedLeafQueue
.getOrderingPolicy() instanceof FairOrderingPolicy);
Expand Down Expand Up @@ -171,6 +171,35 @@ public void testAutoCreateLeafQueueCreation() throws Exception {
}
}

@Test(timeout = 20000)
public void testAutoCreateLeafQueueCreationSchedulerMaximumAllocation()
throws Exception {
try {
// Check the minimum/maximum allocation settings via the
// yarn.scheduler.minimum/maximum-allocation-mb/vcore property
setSchedulerMinMaxAllocation(cs.getConfiguration());
cs.getConfiguration().setAutoCreatedLeafQueueConfigMaximumAllocation(C,
"memory-mb=18384,vcores=8");
cs.reinitialize(cs.getConfiguration(), mockRM.getRMContext());

// submit an app
submitApp(mockRM, cs.getQueue(PARENT_QUEUE), USER0, USER0, 1, 1);

// check preconditions
List<ApplicationAttemptId> appsInC = cs.getAppsInQueue(PARENT_QUEUE);
assertEquals(1, appsInC.size());
assertNotNull(cs.getQueue(USER0));

AutoCreatedLeafQueue autoCreatedLeafQueue =
(AutoCreatedLeafQueue) cs.getQueue(USER0);

validateContainerLimits(autoCreatedLeafQueue, 8, 18384);
} finally {
cleanupQueue(USER0);
cleanupQueue(TEST_GROUPUSER);
}
}

@Test(timeout = 20000)
public void testAutoCreateLeafQueueCreationUsingFullParentPath()
throws Exception {
Expand Down Expand Up @@ -936,7 +965,7 @@ public void testReinitializeQueuesWithAutoCreatedLeafQueues()
validateCapacities(user3Queue, 0.3f, 0.09f, 0.4f,0.2f);

validateUserAndAppLimits(user3Queue, 4000, 4000);
validateContainerLimits(user3Queue);
validateContainerLimits(user3Queue, 6, 10240);

GuaranteedOrZeroCapacityOverTimePolicy autoCreatedQueueManagementPolicy =
(GuaranteedOrZeroCapacityOverTimePolicy) ((ManagedParentQueue)
Expand Down