k8s: allow setting multiple k8s API server addresses

Documentation/operations/upgrade.rst

@@ -365,6 +365,9 @@ New Options
 * ``nodes-gc-interval``: This option was marked as deprecated and has no effect
   in 1.11. Cilium Node Garbage collector is added back in 1.12 (but for k8s GC instead
   of kvstore), so this flag is moved out of deprecated list.
+* ``k8s-api-server-urls``: This option specifies a list of URLs for Kubernetes
+  API server instances. The client will be configured to connect to one of these servers.
+  A new backend server is selected for client connections if the heartbeat check fails.
 
 Removed Options
 ~~~~~~~~~~~~~~~
@@ -393,6 +396,8 @@ Deprecated Options
   Helm) was deprecated, and it will be removed in version 1.13.
 * The ``probe`` option of ``kube-proxy-replacement`` was deprecated, and it will
   be removed in version 1.13.
+* ``k8s-api-server``: This option has been deprecated in favor of ``k8s-api-server-urls``
+  and will be removed in 1.13.
 
 Helm Options
 ~~~~~~~~~~~~
@@ -423,6 +428,8 @@ Helm Options
   container images are not scheduled on non-Linux nodes.
 * ``cluster.id`` cannot be empty and a value must be specified.
   Use the ``0`` value to leave Cluster Mesh disabled.
+* ``k8s.apiServerURLs`` has been introduced to specify multiple Kubernetes API
+  server instances for k8s client configuration.
 
 .. _1.11_upgrade_notes:
 

cilium/cmd/preflight_identity_crd_migrate.go

@@ -181,7 +181,7 @@ func initK8s(ctx context.Context) (crdBackend allocator.Backend, crdAllocator *a
 	k8sClientQPSLimit := viper.GetFloat64(option.K8sClientQPSLimit)
 	k8sClientBurst := viper.GetInt(option.K8sClientBurst)
 
-	k8s.Configure(k8sAPIServer, k8sKubeConfigPath, float32(k8sClientQPSLimit), k8sClientBurst)
+	k8s.Configure([]string{k8sAPIServer}, k8sKubeConfigPath, float32(k8sClientQPSLimit), k8sClientBurst)
 
 	if err := k8s.Init(k8sconfig.NewDefaultConfiguration()); err != nil {
 		log.WithError(err).Fatal("Unable to connect to Kubernetes apiserver")

cilium/cmd/preflight_k8s_valid_cnp.go

@@ -56,7 +56,7 @@ func validateCNPs() error {
 	k8sClientQPSLimit := viper.GetFloat64(option.K8sClientQPSLimit)
 	k8sClientBurst := viper.GetInt(option.K8sClientBurst)
 
-	k8s.Configure(k8sAPIServer, k8sKubeConfigPath, float32(k8sClientQPSLimit), k8sClientBurst)
+	k8s.Configure([]string{k8sAPIServer}, k8sKubeConfigPath, float32(k8sClientQPSLimit), k8sClientBurst)
 
 	if err := k8s.Init(k8sconfig.NewDefaultConfiguration()); err != nil {
 		log.WithError(err).Fatal("Unable to connect to Kubernetes apiserver")

clustermesh-apiserver/main.go

@@ -571,7 +571,7 @@ func runServer(cmd *cobra.Command) {
 	}).Info("Starting clustermesh-apiserver...")
 
 	if mockFile == "" {
-		k8s.Configure("", option.Config.K8sKubeConfigPath, 0.0, 0)
+		k8s.Configure([]string{}, option.Config.K8sKubeConfigPath, 0.0, 0)
 		if err := k8s.Init(k8sconfig.NewDefaultConfiguration()); err != nil {
 			log.WithError(err).Fatal("Unable to connect to Kubernetes apiserver")
 		}

daemon/cmd/config.go

@@ -182,7 +182,7 @@ func (h *getConfig) Handle(params GetConfigParams) middleware.Responder {
 	status := &models.DaemonConfigurationStatus{
 		Addressing:       node.GetNodeAddressing(),
 		K8sConfiguration: k8s.GetKubeconfigPath(),
-		K8sEndpoint:      k8s.GetAPIServerURL(),
+		K8sEndpoint:      k8s.GetAPIServerURLString(),
 		NodeMonitor:      d.monitorAgent.State(),
 		KvstoreConfiguration: &models.KVstoreConfiguration{
 			Type:    option.Config.KVStore,

daemon/cmd/daemon_main.go

@@ -513,6 +513,8 @@ func initializeFlags() {
 
 	flags.String(option.K8sAPIServer, "", "Kubernetes API server URL")
 	option.BindEnv(option.K8sAPIServer)
+	flags.MarkDeprecated(option.K8sAPIServer,
+		fmt.Sprintf("This option is deprecated in favor of %s and will be removed in v1.13", option.K8sAPIServerURLs))
 
 	flags.String(option.K8sKubeConfigPath, "", "Absolute path of the kubernetes kubeconfig file")
 	option.BindEnv(option.K8sKubeConfigPath)
@@ -1146,6 +1148,9 @@ func initializeFlags() {
 	flags.Bool(option.EnableBGPControlPlane, false, "Enable the BGP control plane.")
 	option.BindEnv(option.EnableBGPControlPlane)
 
+	flags.StringSlice(option.K8sAPIServerURLs, []string{}, "List of URLs for Kubernetes API server instances")
+	option.BindEnv(option.K8sAPIServerURLs)
+
 	viper.BindPFlags(flags)
 }
 
@@ -1202,7 +1207,20 @@ func initEnv(cmd *cobra.Command) {
 	// Configure k8s as soon as possible so that k8s.IsEnabled() has the right
 	// behavior.
 	bootstrapStats.k8sInit.Start()
-	k8s.Configure(option.Config.K8sAPIServer, option.Config.K8sKubeConfigPath, defaults.K8sClientQPSLimit, defaults.K8sClientBurst)
+	// The flag K8sAPIServerURLs takes precedence over K8sAPIServer.
+	if len(option.Config.K8sAPIServerURLs) > 0 {
+		k8s.Configure(
+			option.Config.K8sAPIServerURLs,
+			option.Config.K8sKubeConfigPath,
+			defaults.K8sClientQPSLimit,
+			defaults.K8sClientBurst)
+	} else {
+		k8s.Configure(
+			[]string{option.Config.K8sAPIServer},
+			option.Config.K8sKubeConfigPath,
+			defaults.K8sClientQPSLimit,
+			defaults.K8sClientBurst)
+	}
 	bootstrapStats.k8sInit.End(true)
 
 	for _, grp := range option.Config.DebugVerbose {

install/kubernetes/cilium/templates/cilium-configmap.yaml

@@ -675,6 +675,9 @@ data:
 {{- if hasKey .Values.k8s "requireIPv6PodCIDR" }}
   k8s-require-ipv6-pod-cidr: {{ .Values.k8s.requireIPv6PodCIDR | quote }}
 {{- end }}
+{{- if hasKey .Values.k8s "apiServerURLs" }}
+  k8s-api-server-urls: {{ .Values.k8s.apiServerURLs | quote }}
+{{- end }}
 {{- if .Values.endpointStatus.enabled }}
   endpoint-status: {{ required "endpointStatus.status required: policy, health, controllers, logs and / or state. For 2 or more options use a comma: \"policy, health\"" .Values.endpointStatus.status | quote }}
 {{- end }}

install/kubernetes/cilium/values.yaml

@@ -1172,6 +1172,10 @@ k8s: {}
   # range via the Kubernetes node resource
   # requireIPv6PodCIDR: false
 
+  # -- A space separated list of Kubernetes API server URLs to use with the client.
+  # For example "https://192.168.0.1:6443 https://192.168.0.2:6443"
+  # apiServerURLs: ""
+
 # -- Keep the deprecated selector labels when deploying Cilium DaemonSet.
 keepDeprecatedLabels: false
 

install/kubernetes/cilium/values.yaml.tmpl

@@ -1169,6 +1169,10 @@ k8s: {}
   # range via the Kubernetes node resource
   # requireIPv6PodCIDR: false
 
+  # -- A space separated list of Kubernetes API server URLs to use with the client.
+  # For example "https://192.168.0.1:6443 https://192.168.0.2:6443"
+  # apiServerURLs: ""
+
 # -- Keep the deprecated selector labels when deploying Cilium DaemonSet.
 keepDeprecatedLabels: false
 

operator/flags.go

@@ -237,6 +237,11 @@ func init() {
 
 	flags.String(option.K8sAPIServer, "", "Kubernetes API server URL")
 	option.BindEnv(option.K8sAPIServer)
+	flags.MarkDeprecated(option.K8sAPIServer,
+		fmt.Sprintf("This option is deprecated in favor of %s and will be removed in v1.13", option.K8sAPIServerURLs))
+
+	flags.StringSlice(option.K8sAPIServerURLs, []string{}, "List of URLs for Kubernetes API server instances")
+	option.BindEnv(option.K8sAPIServerURLs)
 
 	flags.Float32(option.K8sClientQPSLimit, defaults.K8sClientQPSLimit, "Queries per second limit for the K8s client")
 	flags.Int(option.K8sClientBurst, defaults.K8sClientBurst, "Burst value allowed for the K8s client")

operator/main.go

@@ -137,8 +137,16 @@ func initEnv() {
 }
 
 func initK8s(k8sInitDone chan struct{}) {
+	var apiServerURLs []string
+
+	if len(option.Config.K8sAPIServerURLs) > 0 {
+		apiServerURLs = option.Config.K8sAPIServerURLs
+	} else if option.Config.K8sAPIServer != "" {
+		apiServerURLs = []string{option.Config.K8sAPIServer}
+	}
+
 	k8s.Configure(
-		option.Config.K8sAPIServer,
+		apiServerURLs,
 		option.Config.K8sKubeConfigPath,
 		float32(option.Config.K8sClientQPSLimit),
 		option.Config.K8sClientBurst,

pkg/k8s/client.go

@@ -82,7 +82,7 @@ func WatcherAPIExtClient() *K8sAPIExtensionsClient {
 // CreateConfig creates a client configuration based on the configured API
 // server and Kubeconfig path
 func CreateConfig() (*rest.Config, error) {
-	return createConfig(GetAPIServerURL(), GetKubeconfigPath(), GetQPS(), GetBurst())
+	return createConfig(GetAPIServerURLString(), GetKubeconfigPath(), GetQPS(), GetBurst())
 }
 
 // CreateConfigFromAgentResponse creates a client configuration from a
@@ -109,7 +109,14 @@ func createClient(config *rest.Config, cs kubernetes.Interface) error {
 		case <-timeout.C:
 			log.WithError(err).WithField(logfields.IPAddr, config.Host).Error("Unable to contact k8s api-server")
 			close(stop)
+
 		default:
+			// If we are using explicit connection to API Server
+			// Rotate the URL to see if we can connect to any other instance.
+			if CanRotateAPIServerURL() {
+				RotateAPIServerURL()
+				config.Host = GetAPIServerURL().Host
+			}
 		}
 	}, 5*time.Second, stop)
 	if err == nil {
@@ -271,6 +278,12 @@ func runHeartbeat(heartBeat func(context.Context) error, timeout time.Duration,
 	case err := <-done:
 		if err != nil {
 			log.WithError(err).Warn("Network status error received, restarting client connections")
+
+			if CanRotateAPIServerURL() {
+				RotateAPIServerURL()
+				log.WithField("url", GetAPIServerURLString()).Info("Rotated Kubernetes API server URL for client connections")
+			}
+
 			for _, fn := range closeAllConns {
 				fn()
 			}

pkg/k8s/cnp_test.go

@@ -65,9 +65,9 @@ func (k *K8sIntegrationSuite) SetUpSuite(c *C) {
 	}
 	if os.Getenv("INTEGRATION") != "" {
 		if k8sConfigPath := os.Getenv("KUBECONFIG"); k8sConfigPath == "" {
-			Configure("", "/var/lib/cilium/cilium.kubeconfig", defaults.K8sClientQPSLimit, defaults.K8sClientBurst)
+			Configure([]string{}, "/var/lib/cilium/cilium.kubeconfig", defaults.K8sClientQPSLimit, defaults.K8sClientBurst)
 		} else {
-			Configure("", k8sConfigPath, defaults.K8sClientQPSLimit, defaults.K8sClientBurst)
+			Configure([]string{}, k8sConfigPath, defaults.K8sClientQPSLimit, defaults.K8sClientBurst)
 		}
 		restConfig, err := CreateConfig()
 		c.Assert(err, IsNil)