Skip to content

[SPARK-37908][K8S][TESTS] Refactoring on pod label test in BasicDriver/ExecutorFeatureStepSuite #35209

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Yikun wants to merge 1 commit into from
Closed

[SPARK-37908][K8S][TESTS] Refactoring on pod label test in BasicDriver/ExecutorFeatureStepSuite #35209

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions ...ore/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import scala.collection.JavaConverters._
import io.fabric8.kubernetes.api.model.{ContainerPort, ContainerPortBuilder, LocalObjectReferenceBuilder, Quantity}

import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesTestConf, SparkPod}
import org.apache.spark.deploy.k8s.{KubernetesTestConf, SparkPod}
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
import org.apache.spark.deploy.k8s.features.KubernetesFeaturesTestUtils.TestResourceInformation
Expand All @@ -34,7 +34,7 @@ import org.apache.spark.util.Utils

class BasicDriverFeatureStepSuite extends SparkFunSuite {

private val DRIVER_LABELS = Map("labelkey" -> "labelvalue")
private val CUSTOM_DRIVER_LABELS = Map("labelkey" -> "labelvalue")
private val CONTAINER_IMAGE_PULL_POLICY = "IfNotPresent"
private val DRIVER_ANNOTATIONS = Map("customAnnotation" -> "customAnnotationValue")
private val DRIVER_ENVS = Map(
Expand Down Expand Up @@ -64,7 +64,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
}
val kubernetesConf = KubernetesTestConf.createDriverConf(
sparkConf = sparkConf,
labels = DRIVER_LABELS,
labels = CUSTOM_DRIVER_LABELS,
environment = DRIVER_ENVS,
annotations = DRIVER_ANNOTATIONS)

Expand Down Expand Up @@ -116,12 +116,13 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {

val driverPodMetadata = configuredPod.pod.getMetadata
assert(driverPodMetadata.getName === "spark-driver-pod")
val DEFAULT_LABELS = Map(
SPARK_APP_NAME_LABEL-> KubernetesConf.getAppNameLabel(kubernetesConf.appName)
)
(DRIVER_LABELS ++ DEFAULT_LABELS).foreach { case (k, v) =>

// Check custom and preset labels are as expected
CUSTOM_DRIVER_LABELS.foreach { case (k, v) =>
assert(driverPodMetadata.getLabels.get(k) === v)
}
assert(driverPodMetadata.getLabels === kubernetesConf.labels.asJava)

assert(driverPodMetadata.getAnnotations.asScala === DRIVER_ANNOTATIONS)
assert(configuredPod.pod.getSpec.getRestartPolicy === "Never")
val expectedSparkConf = Map(
Expand Down
15 changes: 8 additions & 7 deletions ...e/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import io.fabric8.kubernetes.api.model._
import org.scalatest.BeforeAndAfter

import org.apache.spark.{SecurityManager, SparkConf, SparkException, SparkFunSuite}
import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesExecutorConf, KubernetesTestConf, SecretVolumeUtils, SparkPod}
import org.apache.spark.deploy.k8s.{KubernetesExecutorConf, KubernetesTestConf, SecretVolumeUtils, SparkPod}
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
import org.apache.spark.deploy.k8s.features.KubernetesFeaturesTestUtils.TestResourceInformation
Expand All @@ -54,7 +54,7 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
private val DRIVER_POD_UID = "driver-uid"
private val RESOURCE_NAME_PREFIX = "base"
private val EXECUTOR_IMAGE = "executor-image"
private val LABELS = Map("label1key" -> "label1value")
private val CUSTOM_EXECUTOR_LABELS = Map("label1key" -> "label1value")
private var defaultProfile: ResourceProfile = _
private val TEST_IMAGE_PULL_SECRETS = Seq("my-1secret-1", "my-secret-2")
private val TEST_IMAGE_PULL_SECRET_OBJECTS =
Expand Down Expand Up @@ -93,7 +93,7 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
KubernetesTestConf.createExecutorConf(
sparkConf = baseConf,
driverPod = Some(DRIVER_POD),
labels = LABELS,
labels = CUSTOM_EXECUTOR_LABELS,
environment = environment)
}

Expand Down Expand Up @@ -156,12 +156,13 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {

// The executor pod name and default labels.
assert(executor.pod.getMetadata.getName === s"$RESOURCE_NAME_PREFIX-exec-1")
val DEFAULT_LABELS = Map(
SPARK_APP_NAME_LABEL-> KubernetesConf.getAppNameLabel(conf.appName)
)
(LABELS ++ DEFAULT_LABELS).foreach { case (k, v) =>

// Check custom and preset labels are as expected
CUSTOM_EXECUTOR_LABELS.foreach { case (k, v) =>
assert(executor.pod.getMetadata.getLabels.get(k) === v)
}
assert(executor.pod.getMetadata.getLabels === conf.labels.asJava)

assert(executor.pod.getSpec.getImagePullSecrets.asScala === TEST_IMAGE_PULL_SECRET_OBJECTS)

// There is exactly 1 container with 1 volume mount and default memory limits.
Expand Down