diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index 6a51973e25f5b..61d6154ccb084 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -507,6 +507,13 @@ See the [configuration page](configuration.html) for information on Spark config configuration properties e.g. spark.kubernetes.namespace. + + spark.kubernetes.driver.master + https://kubernetes.default.svc + + The internal Kubernetes master (API server) address to be used for driver to request executors. + + spark.kubernetes.namespace default diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala index 819ea4a7b13d0..b326650476add 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala @@ -36,6 +36,13 @@ private[spark] object Config extends Logging { .stringConf .createOptional + val KUBERNETES_DRIVER_MASTER_URL = + ConfigBuilder("spark.kubernetes.driver.master") + .doc("The internal Kubernetes master (API server) address " + + "to be used for driver to request executors.") + .stringConf + .createWithDefault(KUBERNETES_MASTER_INTERNAL_URL) + val KUBERNETES_NAMESPACE = ConfigBuilder("spark.kubernetes.namespace") .doc("The namespace that will be used for running the driver and executor pods.") diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index c591214d10796..b9d7a7083f41a 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -51,7 +51,7 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit "If the application is deployed using spark-submit in cluster mode, the driver pod name " + "must be provided.") (KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX, - KUBERNETES_MASTER_INTERNAL_URL, + sc.conf.get(KUBERNETES_DRIVER_MASTER_URL), Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH)), Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_CA_CRT_PATH))) } else {