Merge pull request kubernetes#21535 from AdoHe/restore_secure_etcd

restore ability to run against secured etcd

cmd/kube-apiserver/app/options/options.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/kubernetes/pkg/genericapiserver"
 	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
 	"k8s.io/kubernetes/pkg/master/ports"
+	etcdstorage "k8s.io/kubernetes/pkg/storage/etcd"
 	"k8s.io/kubernetes/pkg/util"
 	utilnet "k8s.io/kubernetes/pkg/util/net"
 
@@ -57,9 +58,8 @@ type APIServer struct {
 	EnableLogsSupport          bool
 	EnableProfiling            bool
 	EnableWatchCache           bool
-	EtcdPathPrefix             string
-	EtcdServerList             []string
 	EtcdServersOverrides       []string
+	EtcdConfig                 etcdstorage.EtcdConfig
 	EventTTL                   time.Duration
 	ExternalHost               string
 	KeystoneURL                string
@@ -100,13 +100,15 @@ func NewAPIServer() *APIServer {
 		AuthorizationMode:       "AlwaysAllow",
 		DeleteCollectionWorkers: 1,
 		EnableLogsSupport:       true,
-		EtcdPathPrefix:          genericapiserver.DefaultEtcdPathPrefix,
-		EventTTL:                1 * time.Hour,
-		MasterCount:             1,
-		MasterServiceNamespace:  api.NamespaceDefault,
-		RuntimeConfig:           make(util.ConfigurationMap),
-		StorageVersions:         registered.AllPreferredGroupVersions(),
-		DefaultStorageVersions:  registered.AllPreferredGroupVersions(),
+		EtcdConfig: etcdstorage.EtcdConfig{
+			Prefix: genericapiserver.DefaultEtcdPathPrefix,
+		},
+		EventTTL:               1 * time.Hour,
+		MasterCount:            1,
+		MasterServiceNamespace: api.NamespaceDefault,
+		RuntimeConfig:          make(util.ConfigurationMap),
+		StorageVersions:        registered.AllPreferredGroupVersions(),
+		DefaultStorageVersions: registered.AllPreferredGroupVersions(),
 		KubeletConfig: kubeletclient.KubeletClientConfig{
 			Port:        ports.KubeletPort,
 			EnableHttps: true,
@@ -220,10 +222,13 @@ func (s *APIServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.AuthorizationConfig.WebhookConfigFile, "authorization-webhook-config-file", s.AuthorizationConfig.WebhookConfigFile, "File with webhook configuration in kubeconfig format, used with --authorization-mode=Webhook. The API server will query the remote service to determine access on the API server's secure port.")
 	fs.StringVar(&s.AdmissionControl, "admission-control", s.AdmissionControl, "Ordered list of plug-ins to do admission control of resources into cluster. Comma-delimited list of: "+strings.Join(admission.GetPlugins(), ", "))
 	fs.StringVar(&s.AdmissionControlConfigFile, "admission-control-config-file", s.AdmissionControlConfigFile, "File with admission control configuration.")
-	fs.StringSliceVar(&s.EtcdServerList, "etcd-servers", s.EtcdServerList, "List of etcd servers to watch (http://ip:port), comma separated. Mutually exclusive with -etcd-config")
+	fs.StringSliceVar(&s.EtcdConfig.ServerList, "etcd-servers", s.EtcdConfig.ServerList, "List of etcd servers to watch (http://ip:port), comma separated. Mutually exclusive with -etcd-config")
 	fs.StringSliceVar(&s.EtcdServersOverrides, "etcd-servers-overrides", s.EtcdServersOverrides, "Per-resource etcd servers overrides, comma separated. The individual override format: group/resource#servers, where servers are http://ip:port, semicolon separated.")
-	fs.StringVar(&s.EtcdPathPrefix, "etcd-prefix", s.EtcdPathPrefix, "The prefix for all resource paths in etcd.")
-	fs.BoolVar(&s.EtcdQuorumRead, "etcd-quorum-read", s.EtcdQuorumRead, "If true, enable quorum read")
+	fs.StringVar(&s.EtcdConfig.Prefix, "etcd-prefix", s.EtcdConfig.Prefix, "The prefix for all resource paths in etcd.")
+	fs.StringVar(&s.EtcdConfig.KeyFile, "etcd-keyfile", s.EtcdConfig.KeyFile, "SSL key file used to secure etcd communication")
+	fs.StringVar(&s.EtcdConfig.CertFile, "etcd-certfile", s.EtcdConfig.CertFile, "SSL certification file used to secure etcd communication")
+	fs.StringVar(&s.EtcdConfig.CAFile, "etcd-cafile", s.EtcdConfig.CAFile, "SSL Certificate Authority file used to secure etcd communication")
+	fs.BoolVar(&s.EtcdConfig.Quorum, "etcd-quorum-read", s.EtcdConfig.Quorum, "If true, enable quorum read")
 	fs.StringSliceVar(&s.CorsAllowedOriginList, "cors-allowed-origins", s.CorsAllowedOriginList, "List of allowed origins for CORS, comma separated.  An allowed origin can be a regular expression to support subdomain matching.  If this list is empty CORS will not be enabled.")
 	fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged, "If true, allow privileged containers.")
 	fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, "A CIDR notation IP range from which to assign service cluster IPs. This must not overlap with any IP ranges assigned to nodes for pods.")

cmd/kube-apiserver/app/server.go

@@ -88,9 +88,9 @@ func verifyClusterIPFlags(s *options.APIServer) {
 }
 
 // For testing.
-type newEtcdFunc func([]string, runtime.NegotiatedSerializer, string, string, string, bool) (storage.Interface, error)
+type newEtcdFunc func(runtime.NegotiatedSerializer, string, string, etcdstorage.EtcdConfig) (storage.Interface, error)
 
-func newEtcd(etcdServerList []string, ns runtime.NegotiatedSerializer, storageGroupVersionString, memoryGroupVersionString, pathPrefix string, quorum bool) (etcdStorage storage.Interface, err error) {
+func newEtcd(ns runtime.NegotiatedSerializer, storageGroupVersionString, memoryGroupVersionString string, etcdConfig etcdstorage.EtcdConfig) (etcdStorage storage.Interface, err error) {
 	if storageGroupVersionString == "" {
 		return etcdStorage, fmt.Errorf("storageVersion is required to create a etcd storage")
 	}
@@ -103,10 +103,8 @@ func newEtcd(etcdServerList []string, ns runtime.NegotiatedSerializer, storageGr
 		return nil, fmt.Errorf("couldn't understand memory version %v: %v", memoryGroupVersionString, err)
 	}
 
-	var storageConfig etcdstorage.EtcdConfig
-	storageConfig.ServerList = etcdServerList
-	storageConfig.Prefix = pathPrefix
-	storageConfig.Quorum = quorum
+	var storageConfig etcdstorage.EtcdStorageConfig
+	storageConfig.Config = etcdConfig
 	s, ok := ns.SerializerForMediaType("application/json", nil)
 	if !ok {
 		return nil, fmt.Errorf("unable to find serializer for JSON")
@@ -128,7 +126,7 @@ func newEtcd(etcdServerList []string, ns runtime.NegotiatedSerializer, storageGr
 }
 
 // parse the value of --etcd-servers-overrides and update given storageDestinations.
-func updateEtcdOverrides(overrides []string, storageVersions map[string]string, prefix string, quorum bool, storageDestinations *genericapiserver.StorageDestinations, newEtcdFn newEtcdFunc) {
+func updateEtcdOverrides(overrides []string, storageVersions map[string]string, etcdConfig etcdstorage.EtcdConfig, storageDestinations *genericapiserver.StorageDestinations, newEtcdFn newEtcdFunc) {
 	if len(overrides) == 0 {
 		return
 	}
@@ -157,11 +155,13 @@ func updateEtcdOverrides(overrides []string, storageVersions map[string]string,
 		}
 
 		servers := strings.Split(tokens[1], ";")
+		overrideEtcdConfig := etcdConfig
+		overrideEtcdConfig.ServerList = servers
 		// Note, internalGV will be wrong for things like batch or
 		// autoscalers, but they shouldn't be using the override
 		// storage.
 		internalGV := apigroup.GroupVersion.Group + "/__internal"
-		etcdOverrideStorage, err := newEtcdFn(servers, api.Codecs, storageVersions[apigroup.GroupVersion.Group], internalGV, prefix, quorum)
+		etcdOverrideStorage, err := newEtcdFn(api.Codecs, storageVersions[apigroup.GroupVersion.Group], internalGV, overrideEtcdConfig)
 		if err != nil {
 			glog.Fatalf("Invalid storage version or misconfigured etcd for %s: %v", tokens[0], err)
 		}
@@ -187,7 +187,7 @@ func Run(s *options.APIServer) error {
 	}
 	glog.Infof("Will report %v as public IP address.", s.AdvertiseAddress)
 
-	if len(s.EtcdServerList) == 0 {
+	if len(s.EtcdConfig.ServerList) == 0 {
 		glog.Fatalf("--etcd-servers must be specified")
 	}
 
@@ -286,7 +286,7 @@ func Run(s *options.APIServer) error {
 	if _, found := storageVersions[legacyV1Group.GroupVersion.Group]; !found {
 		glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.GroupVersion.Group, storageVersions)
 	}
-	etcdStorage, err := newEtcd(s.EtcdServerList, api.Codecs, storageVersions[legacyV1Group.GroupVersion.Group], "/__internal", s.EtcdPathPrefix, s.EtcdQuorumRead)
+	etcdStorage, err := newEtcd(api.Codecs, storageVersions[legacyV1Group.GroupVersion.Group], "/__internal", s.EtcdConfig)
 	if err != nil {
 		glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
 	}
@@ -301,7 +301,7 @@ func Run(s *options.APIServer) error {
 		if _, found := storageVersions[expGroup.GroupVersion.Group]; !found {
 			glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.GroupVersion.Group, storageVersions)
 		}
-		expEtcdStorage, err := newEtcd(s.EtcdServerList, api.Codecs, storageVersions[expGroup.GroupVersion.Group], "extensions/__internal", s.EtcdPathPrefix, s.EtcdQuorumRead)
+		expEtcdStorage, err := newEtcd(api.Codecs, storageVersions[expGroup.GroupVersion.Group], "extensions/__internal", s.EtcdConfig)
 		if err != nil {
 			glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err)
 		}
@@ -337,7 +337,7 @@ func Run(s *options.APIServer) error {
 			glog.Fatalf("The storage version for autoscaling must be either 'autoscaling/v1' or 'extensions/v1beta1'")
 		}
 		glog.Infof("Using %v for autoscaling group storage version", storageGroupVersion)
-		autoscalingEtcdStorage, err := newEtcd(s.EtcdServerList, api.Codecs, storageGroupVersion, "extensions/__internal", s.EtcdPathPrefix, s.EtcdQuorumRead)
+		autoscalingEtcdStorage, err := newEtcd(api.Codecs, storageGroupVersion, "extensions/__internal", s.EtcdConfig)
 		if err != nil {
 			glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err)
 		}
@@ -364,14 +364,14 @@ func Run(s *options.APIServer) error {
 			glog.Fatalf("The storage version for batch must be either 'batch/v1' or 'extensions/v1beta1'")
 		}
 		glog.Infof("Using %v for batch group storage version", storageGroupVersion)
-		batchEtcdStorage, err := newEtcd(s.EtcdServerList, api.Codecs, storageGroupVersion, "extensions/__internal", s.EtcdPathPrefix, s.EtcdQuorumRead)
+		batchEtcdStorage, err := newEtcd(api.Codecs, storageGroupVersion, "extensions/__internal", s.EtcdConfig)
 		if err != nil {
 			glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err)
 		}
 		storageDestinations.AddAPIGroup(batch.GroupName, batchEtcdStorage)
 	}
 
-	updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdPathPrefix, s.EtcdQuorumRead, &storageDestinations, newEtcd)
+	updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdConfig, &storageDestinations, newEtcd)
 
 	n := s.ServiceClusterIPRange
 

cmd/kube-apiserver/app/server_test.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/kubernetes/pkg/genericapiserver"
 	"k8s.io/kubernetes/pkg/runtime"
 	"k8s.io/kubernetes/pkg/storage"
+	etcdstorage "k8s.io/kubernetes/pkg/storage/etcd"
 )
 
 func TestLongRunningRequestRegexp(t *testing.T) {
@@ -100,15 +101,19 @@ func TestUpdateEtcdOverrides(t *testing.T) {
 	}
 
 	for _, test := range testCases {
-		newEtcd := func(serverList []string, _ runtime.NegotiatedSerializer, _, _, _ string, _ bool) (storage.Interface, error) {
-			if !reflect.DeepEqual(test.servers, serverList) {
-				t.Errorf("unexpected server list, expected: %#v, got: %#v", test.servers, serverList)
+		newEtcd := func(_ runtime.NegotiatedSerializer, _, _ string, etcdConfig etcdstorage.EtcdConfig) (storage.Interface, error) {
+			if !reflect.DeepEqual(test.servers, etcdConfig.ServerList) {
+				t.Errorf("unexpected server list, expected: %#v, got: %#v", test.servers, etcdConfig.ServerList)
 			}
 			return nil, nil
 		}
 		storageDestinations := genericapiserver.NewStorageDestinations()
 		override := test.apigroup + "/" + test.resource + "#" + strings.Join(test.servers, ";")
-		updateEtcdOverrides([]string{override}, storageVersions, "", false, &storageDestinations, newEtcd)
+		defaultEtcdConfig := etcdstorage.EtcdConfig{
+			Prefix:     genericapiserver.DefaultEtcdPathPrefix,
+			ServerList: []string{"http://127.0.0.1"},
+		}
+		updateEtcdOverrides([]string{override}, storageVersions, defaultEtcdConfig, &storageDestinations, newEtcd)
 		apigroup, ok := storageDestinations.APIGroups[test.apigroup]
 		if !ok {
 			t.Errorf("apigroup: %s not created", test.apigroup)

docs/admin/kube-apiserver.md

@@ -38,6 +38,9 @@ kube-apiserver
       --cloud-provider="": The provider for cloud services.  Empty string for no provider.
       --cors-allowed-origins=[]: List of allowed origins for CORS, comma separated.  An allowed origin can be a regular expression to support subdomain matching.  If this list is empty CORS will not be enabled.
       --delete-collection-workers=1: Number of workers spawned for DeleteCollection call. These are used to speed up namespace cleanup.
+      --etcd-cafile="": SSL Certificate Authority file used to secure etcd communication
+      --etcd-certfile="": SSL certification file used to secure etcd communication
+      --etcd-keyfile="": SSL key file used to secure etcd communication
       --etcd-prefix="/registry": The prefix for all resource paths in etcd.
       --etcd-quorum-read[=false]: If true, enable quorum read
       --etcd-servers=[]: List of etcd servers to watch (http://ip:port), comma separated. Mutually exclusive with -etcd-config
@@ -83,7 +86,7 @@ kube-apiserver
       --watch-cache-sizes=[]: List of watch cache sizes for every resource (pods, nodes, etc.), comma separated. The individual override format: resource#size, where size is a number. It takes effect when watch-cache is enabled.
 ```
 
-###### Auto generated by spf13/cobra on 24-Feb-2016
+###### Auto generated by spf13/cobra on 6-Mar-2016
 
 
 

examples/apiserver/apiserver.go

@@ -34,9 +34,11 @@ import (
 
 func newStorageDestinations(groupName string, groupMeta *apimachinery.GroupMeta) (*genericapiserver.StorageDestinations, error) {
 	storageDestinations := genericapiserver.NewStorageDestinations()
-	var storageConfig etcdstorage.EtcdConfig
-	storageConfig.ServerList = []string{"http://127.0.0.1:4001"}
-	storageConfig.Prefix = genericapiserver.DefaultEtcdPathPrefix
+	var storageConfig etcdstorage.EtcdStorageConfig
+	storageConfig.Config = etcdstorage.EtcdConfig{
+		Prefix:     genericapiserver.DefaultEtcdPathPrefix,
+		ServerList: []string{"http://127.0.0.1:4001"},
+	}
 	storageConfig.Codec = groupMeta.Codec
 	storageInterface, err := storageConfig.NewStorage()
 	if err != nil {

hack/verify-flags/known-flags.txt

@@ -104,6 +104,9 @@ etcd-quorum-read
 etcd-server
 etcd-servers
 etcd-servers-overrides
+etcd-keyfile
+etcd-certfile
+etcd-cafile
 event-burst
 event-qps
 event-ttl

pkg/genericapiserver/server_run_options.go

@@ -30,7 +30,6 @@ type ServerRunOptions struct {
 	BindAddress          net.IP
 	CertDirectory        string
 	ClientCAFile         string
-	EtcdQuorumRead       bool
 	InsecureBindAddress  net.IP
 	InsecurePort         int
 	LongRunningRequestRE string