From 30c76bddea0bbad3943a2cc5255fbcc071125ec6 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Fri, 7 Jul 2017 10:28:30 -0700 Subject: [PATCH] Stop using insecure serving. --- deploy/addons/addon-manager.yaml | 9 ++++ hack/jenkins/linux_integration_tests_none.sh | 3 +- pkg/localkube/apiserver.go | 10 ++-- pkg/localkube/controller-manager.go | 3 +- pkg/localkube/kubelet.go | 5 +- pkg/localkube/proxy.go | 10 ++-- pkg/localkube/ready.go | 29 +++++++++- pkg/localkube/scheduler.go | 3 +- pkg/localkube/storage_provisioner.go | 13 ++--- pkg/minikube/assets/addons.go | 56 ++++++++++---------- pkg/minikube/assets/vm_assets.go | 35 ++++++++++-- pkg/minikube/cluster/cluster.go | 23 +++++++- pkg/util/constants.go | 3 +- pkg/util/kubeconfig/config.go | 37 +++++++------ 14 files changed, 168 insertions(+), 71 deletions(-) diff --git a/deploy/addons/addon-manager.yaml b/deploy/addons/addon-manager.yaml index e4323cdaf069..be75f9b4281b 100644 --- a/deploy/addons/addon-manager.yaml +++ b/deploy/addons/addon-manager.yaml @@ -26,6 +26,9 @@ spec: containers: - name: kube-addon-manager image: gcr.io/google-containers/kube-addon-manager:v6.4-beta.2 + env: + - name: KUBECONFIG + value: /var/lib/localkube/kubeconfig imagePullPolicy: IfNotPresent resources: requests: @@ -35,7 +38,13 @@ spec: - mountPath: /etc/kubernetes/ name: addons readOnly: true + - mountPath: /var/lib/localkube + name: kubeconfig + readOnly: true volumes: - hostPath: path: /etc/kubernetes/ name: addons + - hostPath: + path: /var/lib/localkube + name: kubeconfig diff --git a/hack/jenkins/linux_integration_tests_none.sh b/hack/jenkins/linux_integration_tests_none.sh index d9d90c8e5d2d..57cac6e2f467 100644 --- a/hack/jenkins/linux_integration_tests_none.sh +++ b/hack/jenkins/linux_integration_tests_none.sh @@ -30,7 +30,8 @@ OS_ARCH="linux-amd64" VM_DRIVER="none" JOB_NAME="Linux-None" EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS --use-vendored-driver" -SUDO_PREFIX="sudo " +SUDO_PREFIX="sudo -E" +export KUBECONFIG="/root/.kube/config" # Download files and set permissions source common.sh diff --git a/pkg/localkube/apiserver.go b/pkg/localkube/apiserver.go index f66a8d1574e1..898aed0d2e03 100644 --- a/pkg/localkube/apiserver.go +++ b/pkg/localkube/apiserver.go @@ -39,8 +39,8 @@ func StartAPIServer(lk LocalkubeServer) func() error { config.SecureServing.BindAddress = lk.APIServerAddress config.SecureServing.BindPort = lk.APIServerPort - config.InsecureServing.BindAddress = lk.APIServerInsecureAddress - config.InsecureServing.BindPort = lk.APIServerInsecurePort + // 0 turns off insecure serving. + config.InsecureServing.BindPort = 0 config.Authentication.ClientCert.ClientCA = lk.GetCAPublicKeyCertPath() @@ -84,7 +84,7 @@ func StartAPIServer(lk LocalkubeServer) func() error { } func readyFunc(lk LocalkubeServer) HealthCheck { - hostport := net.JoinHostPort(lk.APIServerInsecureAddress.String(), strconv.Itoa(lk.APIServerInsecurePort)) - addr := "http://" + path.Join(hostport, "healthz") - return healthCheck(addr) + hostport := net.JoinHostPort("localhost", strconv.Itoa(lk.APIServerPort)) + addr := "https://" + path.Join(hostport, "healthz") + return healthCheck(addr, lk) } diff --git a/pkg/localkube/controller-manager.go b/pkg/localkube/controller-manager.go index f380fa3ce44d..ef642ebaf5d4 100644 --- a/pkg/localkube/controller-manager.go +++ b/pkg/localkube/controller-manager.go @@ -19,6 +19,7 @@ package localkube import ( controllerManager "k8s.io/kubernetes/cmd/kube-controller-manager/app" "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" + "k8s.io/minikube/pkg/util" ) func (lk LocalkubeServer) NewControllerManagerServer() Server { @@ -28,7 +29,7 @@ func (lk LocalkubeServer) NewControllerManagerServer() Server { func StartControllerManagerServer(lk LocalkubeServer) func() error { config := options.NewCMServer() - config.Master = lk.GetAPIServerInsecureURL() + config.Kubeconfig = util.DefaultKubeConfigPath // defaults from command config.DeletingPodsQps = 0.1 diff --git a/pkg/localkube/kubelet.go b/pkg/localkube/kubelet.go index d24d37fe6216..199a041f5dfe 100644 --- a/pkg/localkube/kubelet.go +++ b/pkg/localkube/kubelet.go @@ -17,8 +17,10 @@ limitations under the License. package localkube import ( + "k8s.io/apiserver/pkg/util/flag" kubelet "k8s.io/kubernetes/cmd/kubelet/app" "k8s.io/kubernetes/cmd/kubelet/app/options" + "k8s.io/minikube/pkg/util" ) func (lk LocalkubeServer) NewKubeletServer() Server { @@ -29,7 +31,8 @@ func StartKubeletServer(lk LocalkubeServer) func() error { config := options.NewKubeletServer() // Master details - config.APIServerList = []string{lk.GetAPIServerInsecureURL()} + config.KubeConfig = flag.NewStringFlag(util.DefaultKubeConfigPath) + config.RequireKubeConfig = true // Set containerized based on the flag config.Containerized = lk.Containerized diff --git a/pkg/localkube/proxy.go b/pkg/localkube/proxy.go index 1d37c364faf9..ad1a136949b6 100644 --- a/pkg/localkube/proxy.go +++ b/pkg/localkube/proxy.go @@ -18,6 +18,7 @@ package localkube import ( kubeproxy "k8s.io/kubernetes/cmd/kube-proxy/app" + "k8s.io/minikube/pkg/util" "time" @@ -40,8 +41,9 @@ func StartProxyServer(lk LocalkubeServer) func() error { config := &componentconfig.KubeProxyConfiguration{ OOMScoreAdj: &OOMScoreAdj, ClientConnection: componentconfig.ClientConnectionConfiguration{ - Burst: 10, - QPS: 5, + Burst: 10, + QPS: 5, + KubeConfigFile: util.DefaultKubeConfigPath, }, ConfigSyncPeriod: v1.Duration{Duration: 15 * time.Minute}, IPTables: componentconfig.KubeProxyIPTablesConfiguration{ @@ -49,7 +51,7 @@ func StartProxyServer(lk LocalkubeServer) func() error { SyncPeriod: v1.Duration{Duration: 30 * time.Second}, MinSyncPeriod: v1.Duration{Duration: 5 * time.Second}, }, - BindAddress: lk.APIServerInsecureAddress.String(), + BindAddress: lk.APIServerAddress.String(), Mode: componentconfig.ProxyModeIPTables, FeatureGates: lk.FeatureGates, // Disable the healthz check @@ -60,7 +62,7 @@ func StartProxyServer(lk LocalkubeServer) func() error { return func() error { // Creating this config requires the API Server to be up, so do it in the start function itself. - server, err := kubeproxy.NewProxyServer(config, false, runtime.NewScheme(), lk.GetAPIServerInsecureURL()) + server, err := kubeproxy.NewProxyServer(config, false, runtime.NewScheme(), "") if err != nil { panic(err) } diff --git a/pkg/localkube/ready.go b/pkg/localkube/ready.go index 0e0b71289c76..12479f578eaa 100644 --- a/pkg/localkube/ready.go +++ b/pkg/localkube/ready.go @@ -17,6 +17,8 @@ limitations under the License. package localkube import ( + "crypto/tls" + "crypto/x509" "io/ioutil" "net/http" @@ -25,10 +27,33 @@ import ( type HealthCheck func() bool -func healthCheck(addr string) HealthCheck { +func healthCheck(addr string, lk LocalkubeServer) HealthCheck { return func() bool { glog.Infof("Performing healthcheck on %s\n", addr) - resp, err := http.Get(addr) + + cert, err := tls.LoadX509KeyPair(lk.GetPublicKeyCertPath(), lk.GetPrivateKeyCertPath()) + if err != nil { + glog.Error(err) + return false + } + + // Load CA cert + caCert, err := ioutil.ReadFile(lk.GetCAPublicKeyCertPath()) + if err != nil { + glog.Warning(err) + return false + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: caCertPool, + } + tlsConfig.BuildNameToCertificate() + transport := &http.Transport{TLSClientConfig: tlsConfig} + client := &http.Client{Transport: transport} + + resp, err := client.Get(addr) if err != nil { glog.Errorf("Error performing healthcheck: %s", err) return false diff --git a/pkg/localkube/scheduler.go b/pkg/localkube/scheduler.go index 18d584d8ac73..54b46a568257 100644 --- a/pkg/localkube/scheduler.go +++ b/pkg/localkube/scheduler.go @@ -19,6 +19,7 @@ package localkube import ( scheduler "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app" "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app/options" + "k8s.io/minikube/pkg/util" ) func (lk LocalkubeServer) NewSchedulerServer() Server { @@ -29,7 +30,7 @@ func StartSchedulerServer(lk LocalkubeServer) func() error { config := options.NewSchedulerServer() // master details - config.Master = lk.GetAPIServerInsecureURL() + config.Kubeconfig = util.DefaultKubeConfigPath // defaults from command config.EnableProfiling = true diff --git a/pkg/localkube/storage_provisioner.go b/pkg/localkube/storage_provisioner.go index 6d59dfd1b00a..72ab6f5204c8 100644 --- a/pkg/localkube/storage_provisioner.go +++ b/pkg/localkube/storage_provisioner.go @@ -32,7 +32,8 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api/v1" - "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/minikube/pkg/util" ) const ( @@ -121,12 +122,12 @@ func (lk LocalkubeServer) NewStorageProvisionerServer() Server { func StartStorageProvisioner(lk LocalkubeServer) func() error { - // Create an InClusterConfig and use it to create a client for the controller - // to use to communicate with Kubernetes - config := rest.Config{Host: "http://localhost:8080"} return func() error { - - clientset, err := kubernetes.NewForConfig(&config) + config, err := clientcmd.BuildConfigFromFlags("", util.DefaultKubeConfigPath) + if err != nil { + return err + } + clientset, err := kubernetes.NewForConfig(config) if err != nil { glog.Fatalf("Failed to create client: %v", err) } diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index c689c3e51431..e71d65d52ca4 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -28,12 +28,12 @@ import ( ) type Addon struct { - Assets []*MemoryAsset + Assets []*BinDataAsset enabled bool addonName string } -func NewAddon(assets []*MemoryAsset, enabled bool, addonName string) *Addon { +func NewAddon(assets []*BinDataAsset, enabled bool, addonName string) *Addon { a := &Addon{ Assets: assets, enabled: enabled, @@ -55,107 +55,107 @@ func (a *Addon) IsEnabled() (bool, error) { } var Addons = map[string]*Addon{ - "addon-manager": NewAddon([]*MemoryAsset{ - NewMemoryAsset( + "addon-manager": NewAddon([]*BinDataAsset{ + NewBinDataAsset( "deploy/addons/addon-manager.yaml", "/etc/kubernetes/manifests/", "addon-manager.yaml", "0640"), }, true, "addon-manager"), - "dashboard": NewAddon([]*MemoryAsset{ - NewMemoryAsset( + "dashboard": NewAddon([]*BinDataAsset{ + NewBinDataAsset( "deploy/addons/dashboard/dashboard-rc.yaml", constants.AddonsPath, "dashboard-rc.yaml", "0640"), - NewMemoryAsset( + NewBinDataAsset( "deploy/addons/dashboard/dashboard-svc.yaml", constants.AddonsPath, "dashboard-svc.yaml", "0640"), }, true, "dashboard"), - "default-storageclass": NewAddon([]*MemoryAsset{ - NewMemoryAsset( + "default-storageclass": NewAddon([]*BinDataAsset{ + NewBinDataAsset( "deploy/addons/storageclass/storageclass.yaml", constants.AddonsPath, "storageclass.yaml", "0640"), }, true, "default-storageclass"), - "kube-dns": NewAddon([]*MemoryAsset{ - NewMemoryAsset( + "kube-dns": NewAddon([]*BinDataAsset{ + NewBinDataAsset( "deploy/addons/kube-dns/kube-dns-controller.yaml", constants.AddonsPath, "kube-dns-controller.yaml", "0640"), - NewMemoryAsset( + NewBinDataAsset( "deploy/addons/kube-dns/kube-dns-cm.yaml", constants.AddonsPath, "kube-dns-cm.yaml", "0640"), - NewMemoryAsset( + NewBinDataAsset( "deploy/addons/kube-dns/kube-dns-svc.yaml", constants.AddonsPath, "kube-dns-svc.yaml", "0640"), }, true, "kube-dns"), - "heapster": NewAddon([]*MemoryAsset{ - NewMemoryAsset( + "heapster": NewAddon([]*BinDataAsset{ + NewBinDataAsset( "deploy/addons/heapster/influxGrafana-rc.yaml", constants.AddonsPath, "influxGrafana-rc.yaml", "0640"), - NewMemoryAsset( + NewBinDataAsset( "deploy/addons/heapster/grafana-svc.yaml", constants.AddonsPath, "grafana-svc.yaml", "0640"), - NewMemoryAsset( + NewBinDataAsset( "deploy/addons/heapster/influxdb-svc.yaml", constants.AddonsPath, "influxdb-svc.yaml", "0640"), - NewMemoryAsset( + NewBinDataAsset( "deploy/addons/heapster/heapster-rc.yaml", constants.AddonsPath, "heapster-rc.yaml", "0640"), - NewMemoryAsset( + NewBinDataAsset( "deploy/addons/heapster/heapster-svc.yaml", constants.AddonsPath, "heapster-svc.yaml", "0640"), }, false, "heapster"), - "ingress": NewAddon([]*MemoryAsset{ - NewMemoryAsset( + "ingress": NewAddon([]*BinDataAsset{ + NewBinDataAsset( "deploy/addons/ingress/ingress-configmap.yaml", constants.AddonsPath, "ingress-configmap.yaml", "0640"), - NewMemoryAsset( + NewBinDataAsset( "deploy/addons/ingress/ingress-rc.yaml", constants.AddonsPath, "ingress-rc.yaml", "0640"), - NewMemoryAsset( + NewBinDataAsset( "deploy/addons/ingress/ingress-svc.yaml", constants.AddonsPath, "ingress-svc.yaml", "0640"), }, false, "ingress"), - "registry": NewAddon([]*MemoryAsset{ - NewMemoryAsset( + "registry": NewAddon([]*BinDataAsset{ + NewBinDataAsset( "deploy/addons/registry/registry-rc.yaml", constants.AddonsPath, "registry-rc.yaml", "0640"), - NewMemoryAsset( + NewBinDataAsset( "deploy/addons/registry/registry-svc.yaml", constants.AddonsPath, "registry-svc.yaml", "0640"), }, false, "registry"), - "registry-creds": NewAddon([]*MemoryAsset{ - NewMemoryAsset( + "registry-creds": NewAddon([]*BinDataAsset{ + NewBinDataAsset( "deploy/addons/registry-creds/registry-creds-rc.yaml", constants.AddonsPath, "registry-creds-rc.yaml", diff --git a/pkg/minikube/assets/vm_assets.go b/pkg/minikube/assets/vm_assets.go index d93818cd9913..b0a56df180ab 100644 --- a/pkg/minikube/assets/vm_assets.go +++ b/pkg/minikube/assets/vm_assets.go @@ -106,8 +106,35 @@ type MemoryAsset struct { BaseAsset } -func NewMemoryAsset(assetName, targetDir, targetName, permissions string) *MemoryAsset { +func (m *MemoryAsset) GetLength() int { + return m.Length +} + +func (m *MemoryAsset) Read(p []byte) (int, error) { + return m.reader.Read(p) +} + +func NewMemoryAsset(d []byte, targetDir, targetName, permissions string) *MemoryAsset { m := &MemoryAsset{ + BaseAsset{ + TargetDir: targetDir, + TargetName: targetName, + Permissions: permissions, + }, + } + + m.data = d + m.Length = len(m.data) + m.reader = bytes.NewReader(m.data) + return m +} + +type BinDataAsset struct { + BaseAsset +} + +func NewBinDataAsset(assetName, targetDir, targetName, permissions string) *BinDataAsset { + m := &BinDataAsset{ BaseAsset{ AssetName: assetName, TargetDir: targetDir, @@ -119,7 +146,7 @@ func NewMemoryAsset(assetName, targetDir, targetName, permissions string) *Memor return m } -func (m *MemoryAsset) loadData() error { +func (m *BinDataAsset) loadData() error { contents, err := Asset(m.AssetName) if err != nil { return err @@ -130,11 +157,11 @@ func (m *MemoryAsset) loadData() error { return nil } -func (m *MemoryAsset) GetLength() int { +func (m *BinDataAsset) GetLength() int { return m.Length } -func (m *MemoryAsset) Read(p []byte) (int, error) { +func (m *BinDataAsset) Read(p []byte) (int, error) { return m.reader.Read(p) } diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go index 1bc5dd4efe3f..b414d29a7746 100644 --- a/pkg/minikube/cluster/cluster.go +++ b/pkg/minikube/cluster/cluster.go @@ -37,12 +37,16 @@ import ( "github.com/docker/machine/libmachine/state" "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/tools/clientcmd/api/latest" "k8s.io/minikube/pkg/minikube/assets" cfg "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/sshutil" "k8s.io/minikube/pkg/util" + "k8s.io/minikube/pkg/util/kubeconfig" ) var ( @@ -218,7 +222,7 @@ func UpdateCluster(d drivers.Driver, config KubernetesConfig) error { return errors.Wrap(err, "Error updating localkube from uri") } } else { - localkubeFile = assets.NewMemoryAsset("out/localkube", "/usr/local/bin", "localkube", "0777") + localkubeFile = assets.NewBinDataAsset("out/localkube", "/usr/local/bin", "localkube", "0777") } copyableFiles = append(copyableFiles, localkubeFile) @@ -308,6 +312,23 @@ func SetupCerts(d drivers.Driver, apiServerName string, clusterDnsDomain string) return nil } + kubeCfgSetup := &kubeconfig.KubeConfigSetup{ + ClusterName: cfg.GetMachineName(), + ClusterServerAddress: "https://localhost:8443", + ClientCertificate: filepath.Join(util.DefaultCertPath, "apiserver.crt"), + ClientKey: filepath.Join(util.DefaultCertPath, "apiserver.key"), + CertificateAuthority: filepath.Join(util.DefaultCertPath, "ca.crt"), + KeepContext: false, + } + + kubeCfg := api.NewConfig() + kubeconfig.PopulateKubeConfig(kubeCfgSetup, kubeCfg) + data, err := runtime.Encode(latest.Codec, kubeCfg) + + kubeCfgFile := assets.NewMemoryAsset(data, + util.DefaultLocalkubeDirectory, "kubeconfig", "0644") + copyableFiles = append(copyableFiles, kubeCfgFile) + // transfer files to vm via SSH client, err := sshutil.NewSSHClient(d) if err != nil { diff --git a/pkg/util/constants.go b/pkg/util/constants.go index 86d4a40d34ab..4cb67f4c7776 100644 --- a/pkg/util/constants.go +++ b/pkg/util/constants.go @@ -21,6 +21,7 @@ const ( APIServerPort = 8443 DefaultLocalkubeDirectory = "/var/lib/localkube" DefaultCertPath = DefaultLocalkubeDirectory + "/certs/" + DefaultKubeConfigPath = DefaultLocalkubeDirectory + "/kubeconfig" DefaultServiceClusterIP = "10.0.0.1" DefaultDNSDomain = "cluster.local" DefaultDNSIP = "10.0.0.10" @@ -28,5 +29,5 @@ const ( ) func GetAlternateDNS(domain string) []string { - return []string{"kubernetes.default.svc." + domain, "kubernetes.default.svc", "kubernetes.default", "kubernetes"} + return []string{"kubernetes.default.svc." + domain, "kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"} } diff --git a/pkg/util/kubeconfig/config.go b/pkg/util/kubeconfig/config.go index 197af472acae..981fb963d911 100644 --- a/pkg/util/kubeconfig/config.go +++ b/pkg/util/kubeconfig/config.go @@ -66,43 +66,48 @@ func (k *KubeConfigSetup) GetKubeConfigFile() string { return k.kubeConfigFile.Load().(string) } -// SetupKubeconfig reads config from disk, adds the minikube settings, and writes it back. -// activeContext is true when minikube is the CurrentContext -// If no CurrentContext is set, the given name will be used. -func SetupKubeConfig(cfg *KubeConfigSetup) error { - glog.Infoln("Using kubeconfig: ", cfg.GetKubeConfigFile()) - - // read existing config or create new if does not exist - config, err := ReadConfigOrNew(cfg.GetKubeConfigFile()) - if err != nil { - return err - } - +// PopulateKubeConfig populates an api.Config object. +func PopulateKubeConfig(cfg *KubeConfigSetup, kubecfg *api.Config) { clusterName := cfg.ClusterName cluster := api.NewCluster() cluster.Server = cfg.ClusterServerAddress cluster.CertificateAuthority = cfg.CertificateAuthority - config.Clusters[clusterName] = cluster + kubecfg.Clusters[clusterName] = cluster // user userName := cfg.ClusterName user := api.NewAuthInfo() user.ClientCertificate = cfg.ClientCertificate user.ClientKey = cfg.ClientKey - config.AuthInfos[userName] = user + kubecfg.AuthInfos[userName] = user // context contextName := cfg.ClusterName context := api.NewContext() context.Cluster = cfg.ClusterName context.AuthInfo = userName - config.Contexts[contextName] = context + kubecfg.Contexts[contextName] = context // Only set current context to minikube if the user has not used the keepContext flag if !cfg.KeepContext { - config.CurrentContext = contextName + kubecfg.CurrentContext = cfg.ClusterName + } +} + +// SetupKubeConfig reads config from disk, adds the minikube settings, and writes it back. +// activeContext is true when minikube is the CurrentContext +// If no CurrentContext is set, the given name will be used. +func SetupKubeConfig(cfg *KubeConfigSetup) error { + glog.Infoln("Using kubeconfig: ", cfg.GetKubeConfigFile()) + + // read existing config or create new if does not exist + config, err := ReadConfigOrNew(cfg.GetKubeConfigFile()) + if err != nil { + return err } + PopulateKubeConfig(cfg, config) + // write back to disk if err := WriteConfig(config, cfg.GetKubeConfigFile()); err != nil { return err