From 6ad0da8ab45e95b1bb8e21387551973ed6c689ec Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 15 Dec 2022 10:07:29 -0800 Subject: [PATCH 1/2] [SPARK-41410][K8S][FOLLOWUP] Skip splitSlots and requestNewExecutors in case of 0 snapshot --- .../spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala index 52821a519dd02..621f99230bba6 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala @@ -152,6 +152,7 @@ class ExecutorPodsAllocator( applicationId: String, schedulerBackend: KubernetesClusterSchedulerBackend, snapshots: Seq[ExecutorPodsSnapshot]): Unit = { + logDebug(s"Received ${snapshots.size} snapshots") val k8sKnownExecIds = snapshots.flatMap(_.executorPods.keys).distinct newlyCreatedExecutors --= k8sKnownExecIds schedulerKnownNewlyCreatedExecs --= k8sKnownExecIds @@ -353,7 +354,8 @@ class ExecutorPodsAllocator( } val remainingSlotFromPendingPods = maxPendingPods - totalNotRunningPodCount - if (remainingSlotFromPendingPods > 0 && podsToAllocateWithRpId.size > 0) { + if (remainingSlotFromPendingPods > 0 && podsToAllocateWithRpId.size > 0 && + !(snapshots.isEmpty && podAllocOnPVC && maxPVCs <= PVC_COUNTER.get())) { ExecutorPodsAllocator.splitSlots(podsToAllocateWithRpId, remainingSlotFromPendingPods) .foreach { case ((rpId, podCountForRpId, targetNum), sharedSlotFromPendingPods) => val numMissingPodsForRpId = targetNum - podCountForRpId From 969eaebdad588273e3777ace8a4c2c56db9cf0e1 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 15 Dec 2022 14:10:38 -0800 Subject: [PATCH 2/2] Address comment --- .../spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala index 621f99230bba6..7c92e0ff444dc 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala @@ -353,6 +353,9 @@ class ExecutorPodsAllocator( } } + // Try to request new executors only when there exist remaining slots within the maximum + // number of pending pods and new snapshot arrives in case of waiting for releasing of the + // existing PVCs val remainingSlotFromPendingPods = maxPendingPods - totalNotRunningPodCount if (remainingSlotFromPendingPods > 0 && podsToAllocateWithRpId.size > 0 && !(snapshots.isEmpty && podAllocOnPVC && maxPVCs <= PVC_COUNTER.get())) {