diff --git a/cmd/minikube/cmd/root.go b/cmd/minikube/cmd/root.go index 3dce016fe7f8..34814994c08e 100755 --- a/cmd/minikube/cmd/root.go +++ b/cmd/minikube/cmd/root.go @@ -34,6 +34,7 @@ import ( configCmd "k8s.io/minikube/cmd/minikube/cmd/config" "k8s.io/minikube/cmd/util" "k8s.io/minikube/pkg/minikube/bootstrapper" + "k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm" "k8s.io/minikube/pkg/minikube/bootstrapper/localkube" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" @@ -174,6 +175,11 @@ func GetClusterBootstrapper(api libmachine.API, bootstrapperName string) (bootst if err != nil { return nil, errors.Wrap(err, "getting localkube bootstrapper") } + case bootstrapper.BootstrapperTypeKubeadm: + b, err = kubeadm.NewKubeadmBootstrapper(api) + if err != nil { + return nil, errors.Wrap(err, "getting kubeadm bootstrapper") + } default: return nil, fmt.Errorf("Unknown bootstrapper: %s", bootstrapperName) } diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index cbd6b49afec4..4093c88092b6 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -88,8 +88,11 @@ assumes you have already installed one of the VM drivers: virtualbox/vmwarefusio func runStart(cmd *cobra.Command, args []string) { shouldCacheImages := viper.GetBool(cacheImages) + k8sVersion := viper.GetString(kubernetesVersion) + clusterBootstrapper := viper.GetString(cmdcfg.Bootstrapper) + if shouldCacheImages { - go machine.CacheImagesForBootstrapper(viper.GetString(cmdcfg.Bootstrapper)) + go machine.CacheImagesForBootstrapper(k8sVersion, clusterBootstrapper) } api, err := machine.NewAPIClient() if err != nil { @@ -112,8 +115,8 @@ func runStart(cmd *cobra.Command, args []string) { os.Exit(1) } - if dv := viper.GetString(kubernetesVersion); dv != constants.DefaultKubernetesVersion { - validateK8sVersion(dv) + if k8sVersion != constants.DefaultKubernetesVersion { + validateK8sVersion(k8sVersion) } config := cluster.MachineConfig{ @@ -195,7 +198,7 @@ func runStart(cmd *cobra.Command, args []string) { ShouldLoadCachedImages: shouldCacheImages, } - clusterBootstrapper, err := GetClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper)) + k8sBootstrapper, err := GetClusterBootstrapper(api, clusterBootstrapper) if err != nil { glog.Exitf("Error getting cluster bootstrapper: %s", err) } @@ -211,13 +214,13 @@ func runStart(cmd *cobra.Command, args []string) { } fmt.Println("Moving files into cluster...") - if err := clusterBootstrapper.UpdateCluster(kubernetesConfig); err != nil { + if err := k8sBootstrapper.UpdateCluster(kubernetesConfig); err != nil { glog.Errorln("Error updating cluster: ", err) cmdUtil.MaybeReportErrorAndExit(err) } fmt.Println("Setting up certs...") - if err := clusterBootstrapper.SetupCerts(kubernetesConfig); err != nil { + if err := k8sBootstrapper.SetupCerts(kubernetesConfig); err != nil { glog.Errorln("Error configuring authentication: ", err) cmdUtil.MaybeReportErrorAndExit(err) } @@ -253,12 +256,12 @@ func runStart(cmd *cobra.Command, args []string) { fmt.Println("Starting cluster components...") if !exists { - if err := clusterBootstrapper.StartCluster(kubernetesConfig); err != nil { + if err := k8sBootstrapper.StartCluster(kubernetesConfig); err != nil { glog.Errorln("Error starting cluster: ", err) cmdUtil.MaybeReportErrorAndExit(err) } } else { - if err := clusterBootstrapper.RestartCluster(kubernetesConfig); err != nil { + if err := k8sBootstrapper.RestartCluster(kubernetesConfig); err != nil { glog.Errorln("Error restarting cluster: ", err) cmdUtil.MaybeReportErrorAndExit(err) } diff --git a/hack/jenkins/linux_integration_tests_kvm_alt_kubeadm.sh b/hack/jenkins/linux_integration_tests_kvm_alt_kubeadm.sh new file mode 100644 index 000000000000..a54eff3769f8 --- /dev/null +++ b/hack/jenkins/linux_integration_tests_kvm_alt_kubeadm.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This script runs the integration tests on a Linux machine for the KVM Driver + +# The script expects the following env variables: +# MINIKUBE_LOCATION: GIT_COMMIT from upstream build. +# COMMIT: Actual commit ID from upstream build +# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests +# access_token: The Github API access token. Injected by the Jenkins credential provider. + +set -e + +OS_ARCH="linux-amd64" +VM_DRIVER="kvm2" +JOB_NAME="Linux-KVM-Kubeadm" +EXTRA_ARGS="--bootstrapper=kubeadm" + +# Download files and set permissions +source common.sh diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index 726a7b9b0385..277ace077a6b 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -27,7 +27,7 @@ set -e set +x -for job in "OSX-Virtualbox" "OSX-XHyve" "OSX-Hyperkit" "Linux-Virtualbox" "Linux-KVM" "Linux-KVM-Alt" "Linux-None" "Windows-HyperV"; do +for job in "Linux-KVM-Kubeadm" "OSX-Virtualbox-Kubeadm" "OSX-Virtualbox" "OSX-XHyve" "OSX-Hyperkit" "Linux-Virtualbox" "Linux-KVM" "Linux-KVM-Alt" "Linux-None" "Windows-HyperV"; do target_url="https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${job}.txt" curl "https://api.github.com/repos/kubernetes/minikube/statuses/${ghprbActualCommit}?access_token=$access_token" \ -H "Content-Type: application/json" \ diff --git a/hack/jenkins/osx_integration_tests_virtualbox_kubeadm.sh b/hack/jenkins/osx_integration_tests_virtualbox_kubeadm.sh new file mode 100644 index 000000000000..0b9a9447604c --- /dev/null +++ b/hack/jenkins/osx_integration_tests_virtualbox_kubeadm.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This script runs the integration tests on an OSX machine for the Virtualbox Driver + +# The script expects the following env variables: +# MINIKUBE_LOCATION: GIT_COMMIT from upstream build. +# COMMIT: Actual commit ID from upstream build +# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests +# access_token: The Github API access token. Injected by the Jenkins credential provider. + + +set -e +OS_ARCH="darwin-amd64" +VM_DRIVER="virtualbox" +JOB_NAME="OSX-Virtualbox-Kubeadm" +EXTRA_ARGS="--bootstrapper=kubeadm" + +# Download files and set permissions +source common.sh diff --git a/pkg/minikube/assets/vm_assets.go b/pkg/minikube/assets/vm_assets.go index 65921b311cbc..30cbe587453d 100644 --- a/pkg/minikube/assets/vm_assets.go +++ b/pkg/minikube/assets/vm_assets.go @@ -20,6 +20,7 @@ import ( "bytes" "io" "os" + "path/filepath" "github.com/pkg/errors" ) @@ -63,6 +64,10 @@ type FileAsset struct { BaseAsset } +func NewMemoryAssetTarget(d []byte, targetPath, permissions string) *MemoryAsset { + return NewMemoryAsset(d, filepath.Dir(targetPath), filepath.Base(targetPath), permissions) +} + func NewFileAsset(assetName, targetDir, targetName, permissions string) (*FileAsset, error) { f := &FileAsset{ BaseAsset{ diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index 4ee561d8bbef..d906470267fb 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -48,8 +48,16 @@ type KubernetesConfig struct { const ( BootstrapperTypeLocalkube = "localkube" + BootstrapperTypeKubeadm = "kubeadm" ) -var CachedImagesForBootstrapper = map[string][]string{ - BootstrapperTypeLocalkube: constants.LocalkubeCachedImages, +func GetCachedImageList(version string, bootstrapper string) []string { + switch bootstrapper { + case BootstrapperTypeLocalkube: + return constants.LocalkubeCachedImages + case BootstrapperTypeKubeadm: + return constants.GetKubeadmCachedImages(version) + default: + return []string{} + } } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go new file mode 100644 index 000000000000..925ef77a51f3 --- /dev/null +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -0,0 +1,354 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeadm + +import ( + "bytes" + "crypto" + "fmt" + "html/template" + "os" + "path/filepath" + "strings" + "time" + + "github.com/docker/machine/libmachine" + "github.com/docker/machine/libmachine/state" + download "github.com/jimmidyson/go-download" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" + "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/bootstrapper" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/machine" + "k8s.io/minikube/pkg/minikube/sshutil" + "k8s.io/minikube/pkg/util" +) + +type KubeadmBootstrapper struct { + c bootstrapper.CommandRunner +} + +// TODO(r2d4): template this with bootstrapper.KubernetesConfig +const kubeletSystemdConf = ` +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true" +Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true" +Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local" +Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0" +Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs" +ExecStart= +ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS $KUBELET_EXTRA_ARGS +` + +const kubeletService = ` +[Unit] +Description=kubelet: The Kubernetes Node Agent +Documentation=http://kubernetes.io/docs/ + +[Service] +ExecStart=/usr/bin/kubelet +Restart=always +StartLimitInterval=0 +RestartSec=10 + +[Install] +WantedBy=multi-user.target +` + +const kubeadmConfigTmpl = ` +apiVersion: kubeadm.k8s.io/v1alpha1 +kind: MasterConfiguration +api: + advertiseAddress: {{.AdvertiseAddress}} + bindPort: {{.APIServerPort}} +kubernetesVersion: {{.KubernetesVersion}} +certificatesDir: {{.CertDir}} +networking: + serviceSubnet: {{.ServiceCIDR}} +etcd: + dataDir: {{.EtcdDataDir}} +nodeName: {{.NodeName}} +` + +func NewKubeadmBootstrapper(api libmachine.API) (*KubeadmBootstrapper, error) { + h, err := api.Load(config.GetMachineName()) + if err != nil { + return nil, errors.Wrap(err, "getting api client") + } + var cmd bootstrapper.CommandRunner + // The none driver executes commands directly on the host + if h.Driver.DriverName() == constants.DriverNone { + cmd = &bootstrapper.ExecRunner{} + } else { + client, err := sshutil.NewSSHClient(h.Driver) + if err != nil { + return nil, errors.Wrap(err, "getting ssh client") + } + cmd = bootstrapper.NewSSHRunner(client) + } + return &KubeadmBootstrapper{ + c: cmd, + }, nil +} + +//TODO(r2d4): This should most likely check the health of the apiserver +func (k *KubeadmBootstrapper) GetClusterStatus() (string, error) { + statusCmd := `sudo systemctl is-active kubelet &>/dev/null && echo "Running" || echo "Stopped"` + status, err := k.c.CombinedOutput(statusCmd) + if err != nil { + return "", errors.Wrap(err, "getting status") + } + status = strings.TrimSpace(status) + if status == state.Running.String() || status == state.Stopped.String() { + return status, nil + } + return "", fmt.Errorf("Error: Unrecognized output from ClusterStatus: %s", status) +} + +// TODO(r2d4): Should this aggregate all the logs from the control plane? +// Maybe subcommands for each component? minikube logs apiserver? +func (k *KubeadmBootstrapper) GetClusterLogs(follow bool) (string, error) { + var flags []string + if follow { + flags = append(flags, "-f") + } + logsCommand := fmt.Sprintf("sudo journalctl %s -u kubelet", strings.Join(flags, " ")) + + if follow { + if err := k.c.Run(logsCommand); err != nil { + return "", errors.Wrap(err, "getting shell") + } + } + + logs, err := k.c.CombinedOutput(logsCommand) + if err != nil { + return "", errors.Wrap(err, "getting cluster logs") + } + + return logs, nil +} + +func (k *KubeadmBootstrapper) StartCluster(k8s bootstrapper.KubernetesConfig) error { + // We use --skip-preflight-checks since we have our own custom addons + // that we also stick in /etc/kubernetes/manifests + kubeadmTmpl := "sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks" + t := template.Must(template.New("kubeadmTmpl").Parse(kubeadmTmpl)) + b := bytes.Buffer{} + if err := t.Execute(&b, struct{ KubeadmConfigFile string }{constants.KubeadmConfigFile}); err != nil { + return err + } + + err := k.c.Run(b.String()) + if err != nil { + return errors.Wrapf(err, "kubeadm init error running command: %s", b.String()) + } + + //TODO(r2d4): get rid of global here + master = k8s.NodeName + if err := util.RetryAfter(100, unmarkMaster, time.Millisecond*500); err != nil { + return errors.Wrap(err, "timed out waiting to unmark master") + } + + if err := util.RetryAfter(100, elevateKubeSystemPrivileges, time.Millisecond*500); err != nil { + return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges") + } + + return nil +} + +//TODO(r2d4): Split out into shared function between localkube and kubeadm +func addAddons(files *[]assets.CopyableFile) error { + // add addons to file list + // custom addons + assets.AddMinikubeDirToAssets("addons", constants.AddonsPath, files) + // bundled addons + for addonName, addonBundle := range assets.Addons { + // TODO(r2d4): Kubeadm ignores the kube-dns addon and uses its own. + // expose this in a better way + if addonName == "kube-dns" { + continue + } + if isEnabled, err := addonBundle.IsEnabled(); err == nil && isEnabled { + for _, addon := range addonBundle.Assets { + *files = append(*files, addon) + } + } else if err != nil { + return nil + } + } + + return nil +} + +func (k *KubeadmBootstrapper) RestartCluster(k8s bootstrapper.KubernetesConfig) error { + restoreTmpl := ` + sudo kubeadm alpha phase certs all --config {{.KubeadmConfigFile}} && + sudo /usr/bin/kubeadm alpha phase kubeconfig all --config {{.KubeadmConfigFile}} && + sudo /usr/bin/kubeadm alpha phase controlplane all --config {{.KubeadmConfigFile}} && + sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}} + ` + t := template.Must(template.New("restoreTmpl").Parse(restoreTmpl)) + + opts := struct { + KubeadmConfigFile string + }{ + KubeadmConfigFile: constants.KubeadmConfigFile, + } + + b := bytes.Buffer{} + if err := t.Execute(&b, opts); err != nil { + return err + } + + if err := k.c.Run(b.String()); err != nil { + return errors.Wrapf(err, "running cmd: %s", b.String()) + } + + if err := restartKubeProxy(k8s); err != nil { + return errors.Wrap(err, "restarting kube-proxy") + } + + return nil +} + +func (k *KubeadmBootstrapper) SetupCerts(k8s bootstrapper.KubernetesConfig) error { + return bootstrapper.SetupCerts(k.c, k8s) +} + +func (k *KubeadmBootstrapper) UpdateCluster(cfg bootstrapper.KubernetesConfig) error { + if cfg.ShouldLoadCachedImages { + // Make best effort to load any cached images + go machine.LoadImages(k.c, constants.GetKubeadmCachedImages(cfg.KubernetesVersion), constants.ImageCacheDir) + } + kubeadmCfg, err := k.generateConfig(cfg) + if err != nil { + return errors.Wrap(err, "generating kubeadm cfg") + } + + files := []assets.CopyableFile{ + assets.NewMemoryAssetTarget([]byte(kubeletService), constants.KubeletServiceFile, "0640"), + assets.NewMemoryAssetTarget([]byte(kubeletSystemdConf), constants.KubeletSystemdConfFile, "0640"), + assets.NewMemoryAssetTarget([]byte(kubeadmCfg), constants.KubeadmConfigFile, "0640"), + } + + if err := addAddons(&files); err != nil { + return errors.Wrap(err, "adding addons to copyable files") + } + + for _, f := range files { + if err := k.c.Copy(f); err != nil { + return errors.Wrapf(err, "transferring kubeadm file: %+v", f) + } + } + var g errgroup.Group + for _, bin := range []string{"kubelet", "kubeadm"} { + bin := bin + g.Go(func() error { + path, err := maybeDownloadAndCache(bin, cfg.KubernetesVersion) + if err != nil { + return errors.Wrapf(err, "downloading %s", bin) + } + f, err := assets.NewFileAsset(path, "/usr/bin", bin, "0641") + if err != nil { + return errors.Wrap(err, "making new file asset") + } + if err := k.c.Copy(f); err != nil { + return errors.Wrapf(err, "transferring kubeadm file: %+v", f) + } + return nil + }) + } + if err := g.Wait(); err != nil { + return errors.Wrap(err, "downloading binaries") + } + + err = k.c.Run(` +sudo systemctl daemon-reload && +sudo systemctl enable kubelet && +sudo systemctl start kubelet +`) + if err != nil { + return errors.Wrap(err, "starting kubelet") + } + + return nil +} + +func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) { + t := template.Must(template.New("kubeadmConfigTmpl").Parse(kubeadmConfigTmpl)) + + opts := struct { + CertDir string + ServiceCIDR string + AdvertiseAddress string + APIServerPort int + KubernetesVersion string + EtcdDataDir string + NodeName string + }{ + CertDir: util.DefaultCertPath, + ServiceCIDR: util.DefaultInsecureRegistry, + AdvertiseAddress: k8s.NodeIP, + APIServerPort: util.APIServerPort, + KubernetesVersion: k8s.KubernetesVersion, + EtcdDataDir: "/data", //TODO(r2d4): change to something else persisted + NodeName: k8s.NodeName, + } + + b := bytes.Buffer{} + if err := t.Execute(&b, opts); err != nil { + return "", err + } + + return b.String(), nil +} + +func maybeDownloadAndCache(binary, version string) (string, error) { + targetDir := constants.MakeMiniPath("cache", version) + targetFilepath := filepath.Join(targetDir, binary) + + _, err := os.Stat(targetFilepath) + // If it exists, do no verification and continue + if err == nil { + return targetFilepath, nil + } + if !os.IsNotExist(err) { + return "", errors.Wrapf(err, "stat %s version %s at %s", binary, version, targetDir) + } + + if err = os.MkdirAll(targetDir, 0777); err != nil { + return "", errors.Wrapf(err, "mkdir %s", targetDir) + } + + url := constants.GetKubernetesReleaseURL(binary, version) + options := download.FileOptions{ + Mkdirs: download.MkdirAll, + } + + options.Checksum = constants.GetKubernetesReleaseURLSha1(binary, version) + options.ChecksumHash = crypto.SHA1 + + fmt.Printf("Downloading %s %s\n", binary, version) + if err := download.ToFile(url, targetFilepath, options); err != nil { + return "", errors.Wrapf(err, "Error downloading %s %s", binary, version) + } + fmt.Printf("Finished Downloading %s %s\n", binary, version) + + return targetFilepath, nil +} diff --git a/pkg/minikube/bootstrapper/kubeadm/util.go b/pkg/minikube/bootstrapper/kubeadm/util.go new file mode 100644 index 000000000000..2d67864c3e56 --- /dev/null +++ b/pkg/minikube/bootstrapper/kubeadm/util.go @@ -0,0 +1,192 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeadm + +import ( + "bytes" + "encoding/json" + "html/template" + + "github.com/pkg/errors" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + clientv1 "k8s.io/client-go/pkg/api/v1" + rbacv1beta1 "k8s.io/client-go/pkg/apis/rbac/v1beta1" + "k8s.io/minikube/pkg/minikube/bootstrapper" + "k8s.io/minikube/pkg/minikube/service" + "k8s.io/minikube/pkg/util" +) + +const masterTaint = "node-role.kubernetes.io/master" + +var master = "" + +func unmarkMaster() error { + k8s := service.K8s + client, err := k8s.GetCoreClient() + if err != nil { + return errors.Wrap(err, "getting core client") + } + n, err := client.Nodes().Get(master, v1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "getting node %s", master) + } + + oldData, err := json.Marshal(n) + if err != nil { + return errors.Wrap(err, "json marshalling data before patch") + } + + newTaints := []clientv1.Taint{} + for _, taint := range n.Spec.Taints { + if taint.Key == masterTaint { + continue + } + + newTaints = append(newTaints, taint) + } + n.Spec.Taints = newTaints + + newData, err := json.Marshal(n) + if err != nil { + return errors.Wrapf(err, "json marshalling data after patch") + } + + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, clientv1.Node{}) + if err != nil { + return errors.Wrap(err, "creating strategic patch") + } + + if _, err := client.Nodes().Patch(n.Name, types.StrategicMergePatchType, patchBytes); err != nil { + if apierrs.IsConflict(err) { + return errors.Wrap(err, "strategic patch conflict") + } + return errors.Wrap(err, "applying strategic patch") + } + + return nil +} + +// elevateKubeSystemPrivileges gives the kube-system service account +// cluster admin privileges to work with RBAC. +func elevateKubeSystemPrivileges() error { + k8s := service.K8s + client, err := k8s.GetClientset() + clusterRoleBinding := &rbacv1beta1.ClusterRoleBinding{ + ObjectMeta: v1.ObjectMeta{ + Name: "minikube-rbac", + }, + Subjects: []rbacv1beta1.Subject{ + { + Kind: "ServiceAccount", + Name: "default", + Namespace: "kube-system", + }, + }, + RoleRef: rbacv1beta1.RoleRef{ + Kind: "ClusterRole", + Name: "cluster-admin", + }, + } + + _, err = client.RbacV1beta1().ClusterRoleBindings().Create(clusterRoleBinding) + if err != nil { + return errors.Wrap(err, "creating clusterrolebinding") + } + return nil +} + +const ( + kubeconfigConf = "kubeconfig.conf" + kubeProxyConfigmapTmpl = `apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://{{.AdvertiseAddress}}:{{.APIServerPort}} + name: default +contexts: +- context: + cluster: default + namespace: default + user: default + name: default +current-context: default +users: +- name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +` +) + +func restartKubeProxy(k8s bootstrapper.KubernetesConfig) error { + client, err := util.GetClient() + if err != nil { + return errors.Wrap(err, "getting k8s client") + } + + selector := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-proxy"})) + if err := util.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { + return errors.Wrap(err, "waiting for kube-proxy to be up for configmap update") + } + + cfgMap, err := client.CoreV1().ConfigMaps("kube-system").Get("kube-proxy", metav1.GetOptions{}) + if err != nil { + return errors.Wrap(err, "getting kube-proxy configmap") + } + + t := template.Must(template.New("kubeProxyTmpl").Parse(kubeProxyConfigmapTmpl)) + opts := struct { + AdvertiseAddress string + APIServerPort int + }{ + AdvertiseAddress: k8s.NodeIP, + APIServerPort: util.APIServerPort, + } + + kubeconfig := bytes.Buffer{} + if err := t.Execute(&kubeconfig, opts); err != nil { + return errors.Wrap(err, "executing kube proxy configmap template") + } + + data := map[string]string{ + kubeconfigConf: kubeconfig.String(), + } + + cfgMap.Data = data + if _, err := client.CoreV1().ConfigMaps("kube-system").Update(cfgMap); err != nil { + return errors.Wrap(err, "updating configmap") + } + + pods, err := client.CoreV1().Pods("kube-system").List(metav1.ListOptions{ + LabelSelector: "k8s-app=kube-proxy", + }) + if err != nil { + return errors.Wrap(err, "listing kube-proxy pods") + } + for _, pod := range pods.Items { + if err := client.CoreV1().Pods(pod.Namespace).Delete(pod.Name, &metav1.DeleteOptions{}); err != nil { + return errors.Wrapf(err, "deleting pod %+v", pod) + } + } + + return nil +} diff --git a/pkg/minikube/bootstrapper/localkube/localkube.go b/pkg/minikube/bootstrapper/localkube/localkube.go index 8efa2230e61f..9da544bc2e84 100644 --- a/pkg/minikube/bootstrapper/localkube/localkube.go +++ b/pkg/minikube/bootstrapper/localkube/localkube.go @@ -109,7 +109,6 @@ func (lk *LocalkubeBootstrapper) UpdateCluster(config bootstrapper.KubernetesCon if config.ShouldLoadCachedImages { // Make best effort to load any cached images go machine.LoadImages(lk.cmd, constants.LocalkubeCachedImages, constants.ImageCacheDir) - } copyableFiles := []assets.CopyableFile{} diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index a445e60e7f7d..5fca6a6b53b9 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -130,6 +130,12 @@ const ( LocalkubePIDPath = "/var/run/localkube.pid" ) +const ( + KubeletServiceFile = "/lib/systemd/system/kubelet.service" + KubeletSystemdConfFile = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" + KubeadmConfigFile = "/var/lib/kubeadm.yaml" +) + const ( LocalkubeServicePath = "/usr/lib/systemd/system/localkube.service" LocalkubeRunning = "active" @@ -144,6 +150,17 @@ const ( DefaultMountVersion = "9p2000.u" ) +func GetKubernetesReleaseURL(binaryName, version string) string { + // TODO(r2d4): change this to official releases when the alpha controlplane commands are released. + // We are working with unreleased kubeadm changes at HEAD. + return fmt.Sprintf("https://storage.googleapis.com/minikube-builds/v1.7.3/%s", binaryName) + // return fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%s/bin/linux/amd64/%s", version, binaryName) +} + +func GetKubernetesReleaseURLSha1(binaryName, version string) string { + return fmt.Sprintf("%s.sha1", GetKubernetesReleaseURL(binaryName, version)) +} + const IsMinikubeChildProcess = "IS_MINIKUBE_CHILD_PROCESS" const DriverNone = "none" const FileScheme = "file" @@ -164,4 +181,30 @@ var LocalkubeCachedImages = []string{ "gcr.io/google_containers/pause-amd64:3.0", } +func GetKubeadmCachedImages(version string) []string { + return []string{ + // Dashboard + "gcr.io/google_containers/kubernetes-dashboard-amd64:v1.6.3", + + // Addon Manager + "gcr.io/google-containers/kube-addon-manager:v6.4-beta.2", + + // Pause + "gcr.io/google_containers/pause-amd64:3.0", + + // DNS + "gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.4", + "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.4", + "gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.4", + + // etcd + "gcr.io/google_containers/etcd-amd64:3.0.17", + + "gcr.io/google_containers/kube-proxy-amd64:" + version, + "gcr.io/google_containers/kube-scheduler-amd64:" + version, + "gcr.io/google_containers/kube-controller-manager-amd64:" + version, + "gcr.io/google_containers/kube-apiserver-amd64:" + version, + } +} + var ImageCacheDir = MakeMiniPath("cache", "images") diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index 7c182f58ecb9..2a599bbb0ec1 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -39,12 +39,8 @@ import ( const tempLoadDir = "/tmp" -func CacheImagesForBootstrapper(clusterBootstrapper string) error { - images, ok := bootstrapper.CachedImagesForBootstrapper[clusterBootstrapper] - if !ok { - glog.Infoln("Could not find list of images to cache for bootstrapper %s", clusterBootstrapper) - return nil - } +func CacheImagesForBootstrapper(version string, clusterBootstrapper string) error { + images := bootstrapper.GetCachedImageList(version, clusterBootstrapper) if err := CacheImages(images, constants.ImageCacheDir); err != nil { return errors.Wrapf(err, "Caching images for %s", clusterBootstrapper) diff --git a/pkg/minikube/service/service.go b/pkg/minikube/service/service.go index cf3dc2b63903..04628cf0fc66 100644 --- a/pkg/minikube/service/service.go +++ b/pkg/minikube/service/service.go @@ -42,17 +42,26 @@ import ( type K8sClient interface { GetCoreClient() (corev1.CoreV1Interface, error) + GetClientset() (*kubernetes.Clientset, error) } type K8sClientGetter struct{} -var k8s K8sClient +var K8s K8sClient func init() { - k8s = &K8sClientGetter{} + K8s = &K8sClientGetter{} } -func (*K8sClientGetter) GetCoreClient() (corev1.CoreV1Interface, error) { +func (k *K8sClientGetter) GetCoreClient() (corev1.CoreV1Interface, error) { + client, err := k.GetClientset() + if err != nil { + return nil, errors.Wrap(err, "getting clientset") + } + return client.Core(), nil +} + +func (*K8sClientGetter) GetClientset() (*kubernetes.Clientset, error) { loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() configOverrides := &clientcmd.ConfigOverrides{} kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) @@ -64,7 +73,8 @@ func (*K8sClientGetter) GetCoreClient() (corev1.CoreV1Interface, error) { if err != nil { return nil, errors.Wrap(err, "Error creating new client from kubeConfig.ClientConfig()") } - return client.Core(), nil + + return client, nil } type ServiceURL struct { @@ -88,7 +98,7 @@ func GetServiceURLs(api libmachine.API, namespace string, t *template.Template) return nil, err } - client, err := k8s.GetCoreClient() + client, err := K8s.GetCoreClient() if err != nil { return nil, err } @@ -125,7 +135,7 @@ func GetServiceURLsForService(api libmachine.API, namespace, service string, t * return nil, errors.Wrap(err, "Error getting ip from host") } - client, err := k8s.GetCoreClient() + client, err := K8s.GetCoreClient() if err != nil { return nil, err } @@ -178,7 +188,7 @@ func printURLsForService(c corev1.CoreV1Interface, ip, service, namespace string // CheckService waits for the specified service to be ready by returning an error until the service is up // The check is done by polling the endpoint associated with the service and when the endpoint exists, returning no error->service-online func CheckService(namespace string, service string) error { - client, err := k8s.GetCoreClient() + client, err := K8s.GetCoreClient() if err != nil { return errors.Wrap(err, "Error getting kubernetes client") } @@ -242,7 +252,7 @@ func WaitAndMaybeOpenService(api libmachine.API, namespace string, service strin } func GetServiceListByLabel(namespace string, key string, value string) (*v1.ServiceList, error) { - client, err := k8s.GetCoreClient() + client, err := K8s.GetCoreClient() if err != nil { return &v1.ServiceList{}, &util.RetriableError{Err: err} } @@ -265,7 +275,7 @@ func getServiceListFromServicesByLabel(services corev1.ServiceInterface, key str // CreateSecret creates or modifies secrets func CreateSecret(namespace, name string, dataValues map[string]string, labels map[string]string) error { - client, err := k8s.GetCoreClient() + client, err := K8s.GetCoreClient() if err != nil { return &util.RetriableError{Err: err} } @@ -311,7 +321,7 @@ func CreateSecret(namespace, name string, dataValues map[string]string, labels m // DeleteSecret deletes a secret from a namespace func DeleteSecret(namespace, name string) error { - client, err := k8s.GetCoreClient() + client, err := K8s.GetCoreClient() if err != nil { return &util.RetriableError{Err: err} } diff --git a/pkg/minikube/service/service_test.go b/pkg/minikube/service/service_test.go index bba1f639e445..abaf7bdf23e9 100644 --- a/pkg/minikube/service/service_test.go +++ b/pkg/minikube/service/service_test.go @@ -26,6 +26,7 @@ import ( "github.com/docker/machine/libmachine/host" "github.com/pkg/errors" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/kubernetes/typed/core/v1/fake" "k8s.io/client-go/pkg/api/v1" @@ -43,6 +44,10 @@ func (m *MockClientGetter) GetCoreClient() (corev1.CoreV1Interface, error) { }, nil } +func (m *MockClientGetter) GetClientset() (*kubernetes.Clientset, error) { + return nil, nil +} + type MockCoreClient struct { fake.FakeCoreV1 servicesMap map[string]corev1.ServiceInterface @@ -318,13 +323,13 @@ func TestGetServiceURLs(t *testing.T) { }, } - defer revertK8sClient(k8s) + defer revertK8sClient(K8s) for _, test := range tests { test := test t.Run(test.description, func(t *testing.T) { t.Parallel() - k8s = &MockClientGetter{ + K8s = &MockClientGetter{ servicesMap: serviceNamespaces, } urls, err := GetServiceURLs(test.api, test.namespace, defaultTemplate) @@ -383,11 +388,11 @@ func TestGetServiceURLsForService(t *testing.T) { }, } - defer revertK8sClient(k8s) + defer revertK8sClient(K8s) for _, test := range tests { t.Run(test.description, func(t *testing.T) { t.Parallel() - k8s = &MockClientGetter{ + K8s = &MockClientGetter{ servicesMap: serviceNamespaces, } urls, err := GetServiceURLsForService(test.api, test.namespace, test.service, defaultTemplate) @@ -405,5 +410,5 @@ func TestGetServiceURLsForService(t *testing.T) { } func revertK8sClient(k K8sClient) { - k8s = k + K8s = k } diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 95c8416bf98e..a49ae677c150 100755 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -45,7 +45,11 @@ func TestFunctional(t *testing.T) { t.Run("Addons", testAddons) t.Run("Dashboard", testDashboard) t.Run("ServicesList", testServicesList) - t.Run("Provisioning", testProvisioning) + + // Don't run this test on kubeadm bootstrapper for now. + if !strings.Contains(*args, "--bootstrapper=kubeadm") { + t.Run("Provisioning", testProvisioning) + } if !strings.Contains(minikubeRunner.StartArgs, "--vm-driver=none") { t.Run("EnvVars", testClusterEnv)