From df03689d15efa86984ac55f0f747d39bec67b471 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 27 Jul 2025 14:46:41 -0700 Subject: [PATCH 1/3] [SPARK-52933][K8S] Verify if the executor cpu request exceeds limit --- .../k8s/features/BasicExecutorFeatureStep.scala | 4 ++++ .../k8s/features/BasicExecutorFeatureStepSuite.scala | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala index fcbcd797a8066..78493bb09920f 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala @@ -226,6 +226,10 @@ private[spark] class BasicExecutorFeatureStep( val containerWithLimitCores = if (isDefaultProfile) { executorLimitCores.map { limitCores => val executorCpuLimitQuantity = new Quantity(limitCores) + if (executorCpuLimitQuantity.compareTo(executorCpuQuantity) < 0) { + throw new SparkException( + "The executor cpu request should be less than or equal to cpu limit") + } new ContainerBuilder(executorContainerWithConfVolume) .editResources() .addToLimits("cpu", executorCpuLimitQuantity) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala index 37d28c203000e..681852768485b 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala @@ -120,6 +120,18 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter { assert(error.contains("You must specify an amount for gpu")) } + test("SPARK-52933: Verify if the executor cpu request exceeds limit") { + baseConf.set(KUBERNETES_EXECUTOR_REQUEST_CORES, "2") + baseConf.set(KUBERNETES_EXECUTOR_LIMIT_CORES, "1") + val error = intercept[SparkException] { + initDefaultProfile(baseConf) + val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf), + defaultProfile) + val executor = step.configurePod(SparkPod.initialPod()) + }.getMessage() + assert(error.contains("The executor cpu request should be less than or equal to cpu limit")) + } + test("basic executor pod with resources") { val fpgaResourceID = new ResourceID(SPARK_EXECUTOR_PREFIX, FPGA) val gpuExecutorResourceID = new ResourceID(SPARK_EXECUTOR_PREFIX, GPU) From 202dce7da30ab371d5f42675c7c644fee3d70e61 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 27 Jul 2025 16:07:06 -0700 Subject: [PATCH 2/3] indentation --- .../deploy/k8s/features/BasicExecutorFeatureStepSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala index 681852768485b..a23916d33b619 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala @@ -126,7 +126,7 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter { val error = intercept[SparkException] { initDefaultProfile(baseConf) val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf), - defaultProfile) + defaultProfile) val executor = step.configurePod(SparkPod.initialPod()) }.getMessage() assert(error.contains("The executor cpu request should be less than or equal to cpu limit")) From 34c1d39aa427b76710a14fc6c4c9158f1a3f3eac Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 27 Jul 2025 20:14:04 -0700 Subject: [PATCH 3/3] Address comments --- .../spark/deploy/k8s/features/BasicExecutorFeatureStep.scala | 4 ++-- .../deploy/k8s/features/BasicExecutorFeatureStepSuite.scala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala index 78493bb09920f..bd7e7174cff1d 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala @@ -227,8 +227,8 @@ private[spark] class BasicExecutorFeatureStep( executorLimitCores.map { limitCores => val executorCpuLimitQuantity = new Quantity(limitCores) if (executorCpuLimitQuantity.compareTo(executorCpuQuantity) < 0) { - throw new SparkException( - "The executor cpu request should be less than or equal to cpu limit") + throw new SparkException(s"The executor cpu request ($executorCpuQuantity) should be " + + s"less than or equal to cpu limit ($executorCpuLimitQuantity)") } new ContainerBuilder(executorContainerWithConfVolume) .editResources() diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala index a23916d33b619..906727e49276e 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala @@ -129,7 +129,7 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter { defaultProfile) val executor = step.configurePod(SparkPod.initialPod()) }.getMessage() - assert(error.contains("The executor cpu request should be less than or equal to cpu limit")) + assert(error.contains("cpu request (2) should be less than or equal to cpu limit (1)")) } test("basic executor pod with resources") {