diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index 3453ee912205f..e23f28cf755c3 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -334,6 +334,16 @@ the Spark application. ## Kubernetes Features +### Configuration File + +Your Kubernetes config file typically lives under `.kube/config` in your home directory or in a location specified by the `KUBECONFIG` environment variable. Spark on Kubernetes will attempt to use this file to do an initial auto-configuration of the Kubernetes client used to interact with the Kubernetes cluster. A variety of Spark configuration properties are provided that allow further customising the client configuration e.g. using an alternative authentication method. + +### Contexts + +Kubernetes configuration files can contain multiple contexts that allow for switching between different clusters and/or user identities. By default Spark on Kubernetes will use your current context (which can be checked by running `kubectl config current-context`) when doing the initial auto-configuration of the Kubernetes client. + +In order to use an alternative context users can specify the desired context via the Spark configuration property `spark.kubernetes.context` e.g. `spark.kubernetes.context=minikube`. + ### Namespaces Kubernetes has the concept of [namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/). @@ -406,13 +416,23 @@ Some of these include: # Configuration -See the [configuration page](configuration.html) for information on Spark configurations. The following configurations are -specific to Spark on Kubernetes. +See the [configuration page](configuration.html) for information on Spark configurations. The following configurations are specific to Spark on Kubernetes. #### Spark Properties + + + + + diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala index e8bf16df190e8..4cca1e22bd108 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala @@ -24,6 +24,18 @@ import org.apache.spark.internal.config.ConfigBuilder private[spark] object Config extends Logging { + val KUBERNETES_CONTEXT = + ConfigBuilder("spark.kubernetes.context") + .doc("The desired context from your K8S config file used to configure the K8S " + + "client for interacting with the cluster. Useful if your config file has " + + "multiple clusters or user identities defined. The client library used " + + "locates the config file via the KUBECONFIG environment variable or by defaulting " + + "to .kube/config under your home directory. If not specified then your current " + + "context is used. You can always override specific aspects of the config file " + + "provided configuration using other Spark on K8S configuration options.") + .stringConf + .createOptional + val KUBERNETES_NAMESPACE = ConfigBuilder("spark.kubernetes.namespace") .doc("The namespace that will be used for running the driver and executor pods.") diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala index 77bd66b608e7c..06dea42cc135c 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala @@ -21,11 +21,13 @@ import java.io.File import com.google.common.base.Charsets import com.google.common.io.Files import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient, KubernetesClient} +import io.fabric8.kubernetes.client.Config.autoConfigure import io.fabric8.kubernetes.client.utils.HttpClientUtils import okhttp3.Dispatcher import org.apache.spark.SparkConf import org.apache.spark.deploy.k8s.Config._ +import org.apache.spark.internal.Logging import org.apache.spark.util.ThreadUtils /** @@ -33,7 +35,7 @@ import org.apache.spark.util.ThreadUtils * parse configuration keys, similar to the manner in which Spark's SecurityManager parses SSL * options for different components. */ -private[spark] object SparkKubernetesClientFactory { +private[spark] object SparkKubernetesClientFactory extends Logging { def createKubernetesClient( master: String, @@ -42,9 +44,6 @@ private[spark] object SparkKubernetesClientFactory { sparkConf: SparkConf, defaultServiceAccountToken: Option[File], defaultServiceAccountCaCert: Option[File]): KubernetesClient = { - - // TODO [SPARK-25887] Support configurable context - val oauthTokenFileConf = s"$kubernetesAuthConfPrefix.$OAUTH_TOKEN_FILE_CONF_SUFFIX" val oauthTokenConf = s"$kubernetesAuthConfPrefix.$OAUTH_TOKEN_CONF_SUFFIX" val oauthTokenFile = sparkConf.getOption(oauthTokenFileConf) @@ -67,8 +66,16 @@ private[spark] object SparkKubernetesClientFactory { val dispatcher = new Dispatcher( ThreadUtils.newDaemonCachedThreadPool("kubernetes-dispatcher")) - // TODO [SPARK-25887] Create builder in a way that respects configurable context - val config = new ConfigBuilder() + // Allow for specifying a context used to auto-configure from the users K8S config file + val kubeContext = sparkConf.get(KUBERNETES_CONTEXT).filter(_.nonEmpty) + logInfo("Auto-configuring K8S client using " + + kubeContext.map("context " + _).getOrElse("current context") + + " from users K8S config file") + + // Start from an auto-configured config with the desired context + // Fabric 8 uses null to indicate that the users current context should be used so if no + // explicit setting pass null + val config = new ConfigBuilder(autoConfigure(kubeContext.getOrElse(null))) .withApiVersion("v1") .withMasterUrl(master) .withWebsocketPingInterval(0)
Property NameDefaultMeaning
spark.kubernetes.context(none) + The context from the user Kubernetes configuration file used for the initial + auto-configuration of the Kubernetes client library. When not specified then + the users current context is used. NB: Many of the + auto-configured settings can be overridden by the use of other Spark + configuration properties e.g. spark.kubernetes.namespace. +
spark.kubernetes.namespace default