From 96c6bbc1a4df251b4b2c3c8c41732eb8aebdcd92 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Fri, 11 Feb 2022 09:27:17 +0800 Subject: [PATCH 1/4] add doc --- ...ernetesDriverCustomFeatureConfigStep.scala | 33 +++++++++++++++++++ ...netesExecutorCustomFeatureConfigStep.scala | 33 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesDriverCustomFeatureConfigStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesDriverCustomFeatureConfigStep.scala index bbd05e9f67c51..082bd7d748278 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesDriverCustomFeatureConfigStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesDriverCustomFeatureConfigStep.scala @@ -25,6 +25,39 @@ import org.apache.spark.deploy.k8s.KubernetesDriverConf * A base interface to help user extend custom feature step in driver side. * Note: If your custom feature step would be used only in driver or both in driver and executor, * please use this. + * + * Here is an example of driver feature step: + * + * {{{ + * class DriverExampleFeatureStep extends KubernetesDriverCustomFeatureConfigStep { + * private var driverConf: KubernetesDriverConf = _ + * + * override def init(conf: KubernetesDriverConf): Unit = { + * driverConf = conf + * } + * + * override def configurePod(pod: SparkPod): SparkPod = {} + * } + * }}} + * + * Here is an example of feature step both for driver and executor: + * + * {{{ + * class ExecutorExampleFeatureStep extends KubernetesDriverCustomFeatureConfigStep + * with KubernetesExecutorCustomFeatureConfigStep { + * private var kubernetesConf: KubernetesConf = _ + * + * override def init(conf: KubernetesDriverConf): Unit = { + * kubernetesConf = conf + * } + * + * override def init(conf: KubernetesExecutorConf): Unit = { + * kubernetesConf = conf + * } + * + * override def configurePod(pod: SparkPod): SparkPod = {} + * } + * }}} */ @Unstable @DeveloperApi diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala index 062fa7dbf1413..587937fe8de9f 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala @@ -25,6 +25,39 @@ import org.apache.spark.deploy.k8s.KubernetesExecutorConf * A base interface to help user extend custom feature step in executor side. * Note: If your custom feature step would be used only in driver or both in driver and executor, * please use this. + * + * Here is an example of feature step for executor: + * + * {{{ + * class ExecutorExampleFeatureStep extends KubernetesExecutorCustomFeatureConfigStep { + * private var executorConf: KubernetesExecutorConf = _ + * + * override def init(conf: KubernetesExecutorConf): Unit = { + * executorConf = conf + * } + * + * override def configurePod(pod: SparkPod): SparkPod = {} + * } + * }}} + * + * Here is an example of feature step both for driver and executor: + * + * {{{ + * class ExecutorExampleFeatureStep extends KubernetesDriverCustomFeatureConfigStep + * with KubernetesExecutorCustomFeatureConfigStep { + * private var kubernetesConf: KubernetesConf = _ + * + * override def init(conf: KubernetesDriverConf): Unit = { + * kubernetesConf = conf + * } + * + * override def init(conf: KubernetesExecutorConf): Unit = { + * kubernetesConf = conf + * } + * + * override def configurePod(pod: SparkPod): SparkPod = {} + * } + * }}} */ @Unstable @DeveloperApi From 79a1c529d72e1b2227cfa806347ca8a828d38d87 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Sun, 13 Feb 2022 11:40:19 +0800 Subject: [PATCH 2/4] address comments --- ...KubernetesDriverCustomFeatureConfigStep.scala | 16 +++++++++++----- ...bernetesExecutorCustomFeatureConfigStep.scala | 16 +++++++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesDriverCustomFeatureConfigStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesDriverCustomFeatureConfigStep.scala index 082bd7d748278..39dca5d16c951 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesDriverCustomFeatureConfigStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesDriverCustomFeatureConfigStep.scala @@ -26,7 +26,7 @@ import org.apache.spark.deploy.k8s.KubernetesDriverConf * Note: If your custom feature step would be used only in driver or both in driver and executor, * please use this. * - * Here is an example of driver feature step: + * Example of driver feature step: * * {{{ * class DriverExampleFeatureStep extends KubernetesDriverCustomFeatureConfigStep { @@ -36,15 +36,18 @@ import org.apache.spark.deploy.k8s.KubernetesDriverConf * driverConf = conf * } * - * override def configurePod(pod: SparkPod): SparkPod = {} + * // Implements methods of `KubernetesFeatureConfigStep`, such as `configurePod` + * override def configurePod(pod: SparkPod): SparkPod = { + * // Apply modifications on the given pod in accordance to this feature. + * } * } * }}} * - * Here is an example of feature step both for driver and executor: + * Example of feature step for both driver and executor: * * {{{ * class ExecutorExampleFeatureStep extends KubernetesDriverCustomFeatureConfigStep - * with KubernetesExecutorCustomFeatureConfigStep { + * with KubernetesExecutorCustomFeatureConfigStep { * private var kubernetesConf: KubernetesConf = _ * * override def init(conf: KubernetesDriverConf): Unit = { @@ -55,7 +58,10 @@ import org.apache.spark.deploy.k8s.KubernetesDriverConf * kubernetesConf = conf * } * - * override def configurePod(pod: SparkPod): SparkPod = {} + * // Implements methods of `KubernetesFeatureConfigStep`, such as `configurePod` + * override def configurePod(pod: SparkPod): SparkPod = { + * // Apply modifications on the given pod in accordance to this feature. + * } * } * }}} */ diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala index 587937fe8de9f..4f1b6e0b4235b 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala @@ -26,7 +26,7 @@ import org.apache.spark.deploy.k8s.KubernetesExecutorConf * Note: If your custom feature step would be used only in driver or both in driver and executor, * please use this. * - * Here is an example of feature step for executor: + * Example of feature step for both driver and executor: * * {{{ * class ExecutorExampleFeatureStep extends KubernetesExecutorCustomFeatureConfigStep { @@ -36,15 +36,18 @@ import org.apache.spark.deploy.k8s.KubernetesExecutorConf * executorConf = conf * } * - * override def configurePod(pod: SparkPod): SparkPod = {} + * // Implements methods of `KubernetesFeatureConfigStep`, such as `configurePod` + * override def configurePod(pod: SparkPod): SparkPod = { + * // Apply modifications on the given pod in accordance to this feature. + * } * } * }}} * - * Here is an example of feature step both for driver and executor: + * Example of feature step for both driver and executor: * * {{{ * class ExecutorExampleFeatureStep extends KubernetesDriverCustomFeatureConfigStep - * with KubernetesExecutorCustomFeatureConfigStep { + * with KubernetesExecutorCustomFeatureConfigStep { * private var kubernetesConf: KubernetesConf = _ * * override def init(conf: KubernetesDriverConf): Unit = { @@ -55,7 +58,10 @@ import org.apache.spark.deploy.k8s.KubernetesExecutorConf * kubernetesConf = conf * } * - * override def configurePod(pod: SparkPod): SparkPod = {} + * // Implements methods of `KubernetesFeatureConfigStep`, such as `configurePod` + * override def configurePod(pod: SparkPod): SparkPod = { + * // Apply modifications on the given pod in accordance to this feature. + * } * } * }}} */ From 9f194a1545f7c3c8b002866be0eb096bf90c0650 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Sun, 13 Feb 2022 22:45:06 +0800 Subject: [PATCH 3/4] Fix typo --- .../features/KubernetesExecutorCustomFeatureConfigStep.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala index 4f1b6e0b4235b..f0195876c7923 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala @@ -26,7 +26,7 @@ import org.apache.spark.deploy.k8s.KubernetesExecutorConf * Note: If your custom feature step would be used only in driver or both in driver and executor, * please use this. * - * Example of feature step for both driver and executor: + * Example of executor feature step: * * {{{ * class ExecutorExampleFeatureStep extends KubernetesExecutorCustomFeatureConfigStep { From a9650f40856b63d154a5d92ca471830cc87e5935 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Wed, 16 Feb 2022 08:54:18 +0800 Subject: [PATCH 4/4] Address nits --- .../features/KubernetesDriverCustomFeatureConfigStep.scala | 4 ++-- .../features/KubernetesExecutorCustomFeatureConfigStep.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesDriverCustomFeatureConfigStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesDriverCustomFeatureConfigStep.scala index 39dca5d16c951..0edd94d3370ab 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesDriverCustomFeatureConfigStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesDriverCustomFeatureConfigStep.scala @@ -46,8 +46,8 @@ import org.apache.spark.deploy.k8s.KubernetesDriverConf * Example of feature step for both driver and executor: * * {{{ - * class ExecutorExampleFeatureStep extends KubernetesDriverCustomFeatureConfigStep - * with KubernetesExecutorCustomFeatureConfigStep { + * class DriverAndExecutorExampleFeatureStep extends KubernetesDriverCustomFeatureConfigStep + * with KubernetesExecutorCustomFeatureConfigStep { * private var kubernetesConf: KubernetesConf = _ * * override def init(conf: KubernetesDriverConf): Unit = { diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala index f0195876c7923..dfb1c768c990e 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/KubernetesExecutorCustomFeatureConfigStep.scala @@ -46,8 +46,8 @@ import org.apache.spark.deploy.k8s.KubernetesExecutorConf * Example of feature step for both driver and executor: * * {{{ - * class ExecutorExampleFeatureStep extends KubernetesDriverCustomFeatureConfigStep - * with KubernetesExecutorCustomFeatureConfigStep { + * class DriverAndExecutorExampleFeatureStep extends KubernetesDriverCustomFeatureConfigStep + * with KubernetesExecutorCustomFeatureConfigStep { * private var kubernetesConf: KubernetesConf = _ * * override def init(conf: KubernetesDriverConf): Unit = {