From 7cbe253ef6b1591693f746ee403b54807e0ad75a Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Mon, 18 Feb 2019 01:00:12 +0800 Subject: [PATCH 1/6] Allow specifying image repository for docker images Some users (especially for those in mainland China) may have issue accessing the default image repository. This patchset allows users to override the default image repository gcr.io to a different repository by specifying --image-repository option in the command line as a simple workaround. Images will be pulled from the specified image repository instead of the default ones. Example (using mirror by Aliyun): minikube start ... --image-repository registry.cn-hangzhou.aliyuncs.com/google_containers --- cmd/minikube/cmd/start.go | 5 +- deploy/addons/addon-manager.yaml | 2 +- deploy/addons/dashboard/dashboard-dp.yaml | 2 +- .../storage-provisioner.yaml | 2 +- pkg/minikube/assets/addons.go | 114 ++++++++++++------ pkg/minikube/assets/vm_assets.go | 50 +++++++- pkg/minikube/bootstrapper/bootstrapper.go | 5 +- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 49 +++++--- .../bootstrapper/kubeadm/kubeadm_test.go | 2 +- .../bootstrapper/kubeadm/templates.go | 8 +- pkg/minikube/config/types.go | 1 + pkg/minikube/constants/constants.go | 111 ++++++++++------- pkg/minikube/machine/cache_images.go | 4 +- 13 files changed, 241 insertions(+), 114 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 75e2b001280d..a9be0d00d862 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -77,6 +77,7 @@ const ( apiServerPort = "apiserver-port" dnsDomain = "dns-domain" serviceCIDR = "service-cluster-ip-range" + imageRepository = "image-repository" mountString = "mount-string" disableDriverMounts = "disable-driver-mounts" cacheImages = "cache-images" @@ -123,6 +124,7 @@ func init() { startCmd.Flags().String(serviceCIDR, pkgutil.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.") startCmd.Flags().StringSliceVar(&insecureRegistry, "insecure-registry", nil, "Insecure Docker registries to pass to the Docker daemon. The default service CIDR range will automatically be added.") startCmd.Flags().StringSliceVar(®istryMirror, "registry-mirror", nil, "Registry mirrors to pass to the Docker daemon") + startCmd.Flags().String(imageRepository, "", "Image repository to pull down docker images.") startCmd.Flags().String(containerRuntime, "docker", "The container runtime to be used (docker, crio, containerd)") startCmd.Flags().String(criSocket, "", "The cri socket path to be used") startCmd.Flags().String(kubernetesVersion, constants.DefaultKubernetesVersion, "The kubernetes version that the minikube VM will use (ex: v1.2.3)") @@ -237,7 +239,7 @@ func beginCacheImages(g *errgroup.Group, kVersion string) { } console.OutStyle("caching", "Caching images in the background ...") g.Go(func() error { - return machine.CacheImagesForBootstrapper(kVersion, viper.GetString(cmdcfg.Bootstrapper)) + return machine.CacheImagesForBootstrapper(viper.GetString(imageRepository), kVersion, viper.GetString(cmdcfg.Bootstrapper)) }) } @@ -296,6 +298,7 @@ func generateConfig(cmd *cobra.Command, kVersion string) (cfg.Config, error) { CRISocket: viper.GetString(criSocket), NetworkPlugin: selectedNetworkPlugin, ServiceCIDR: viper.GetString(serviceCIDR), + ImageRepository: viper.GetString(imageRepository), ExtraOptions: extraOptions, ShouldLoadCachedImages: viper.GetBool(cacheImages), EnableDefaultCNI: selectedEnableDefaultCNI, diff --git a/deploy/addons/addon-manager.yaml b/deploy/addons/addon-manager.yaml index 105bfd76b828..56f6bb036df9 100644 --- a/deploy/addons/addon-manager.yaml +++ b/deploy/addons/addon-manager.yaml @@ -25,7 +25,7 @@ spec: hostNetwork: true containers: - name: kube-addon-manager - image: k8s.gcr.io/kube-addon-manager:v8.6 + image: {{default "k8s.gcr.io" .ImageRepository}}/kube-addon-manager:v8.6 env: - name: KUBECONFIG value: /var/lib/minikube/kubeconfig diff --git a/deploy/addons/dashboard/dashboard-dp.yaml b/deploy/addons/dashboard/dashboard-dp.yaml index b9b49179849d..6bf897b625b4 100644 --- a/deploy/addons/dashboard/dashboard-dp.yaml +++ b/deploy/addons/dashboard/dashboard-dp.yaml @@ -37,7 +37,7 @@ spec: spec: containers: - name: kubernetes-dashboard - image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1 + image: {{default "k8s.gcr.io" .ImageRepository}}/kubernetes-dashboard-amd64:v1.10.1 imagePullPolicy: IfNotPresent ports: - containerPort: 9090 diff --git a/deploy/addons/storage-provisioner/storage-provisioner.yaml b/deploy/addons/storage-provisioner/storage-provisioner.yaml index 72ee7feefb34..4832acb4cfe6 100644 --- a/deploy/addons/storage-provisioner/storage-provisioner.yaml +++ b/deploy/addons/storage-provisioner/storage-provisioner.yaml @@ -51,7 +51,7 @@ spec: hostNetwork: true containers: - name: storage-provisioner - image: gcr.io/k8s-minikube/storage-provisioner:v1.8.1 + image: {{default "gcr.io/k8s-minikube" .ImageRepository}}/storage-provisioner:v1.8.1 command: ["/storage-provisioner"] imagePullPolicy: IfNotPresent volumeMounts: diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index b6db690e76fa..e21a5aba1f58 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -62,222 +62,260 @@ var Addons = map[string]*Addon{ "deploy/addons/addon-manager.yaml", "/etc/kubernetes/manifests/", "addon-manager.yaml", - "0640"), + "0640", + true), }, true, "addon-manager"), "dashboard": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/dashboard/dashboard-dp.yaml", constants.AddonsPath, "dashboard-dp.yaml", - "0640"), + "0640", + true), NewBinDataAsset( "deploy/addons/dashboard/dashboard-svc.yaml", constants.AddonsPath, "dashboard-svc.yaml", - "0640"), + "0640", + false), }, false, "dashboard"), "default-storageclass": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/storageclass/storageclass.yaml", constants.AddonsPath, "storageclass.yaml", - "0640"), + "0640", + false), }, true, "default-storageclass"), "storage-provisioner": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/storage-provisioner/storage-provisioner.yaml", constants.AddonsPath, "storage-provisioner.yaml", - "0640"), + "0640", + true), }, true, "storage-provisioner"), "storage-provisioner-gluster": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/storage-provisioner-gluster/storage-gluster-ns.yaml", constants.AddonsPath, "storage-gluster-ns.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/storage-provisioner-gluster/glusterfs-daemonset.yaml", constants.AddonsPath, "glusterfs-daemonset.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/storage-provisioner-gluster/heketi-deployment.yaml", constants.AddonsPath, "heketi-deployment.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/storage-provisioner-gluster/storage-provisioner-glusterfile.yaml", constants.AddonsPath, "storage-privisioner-glusterfile.yaml", - "0640"), + "0640", + false), }, false, "storage-provisioner-gluster"), "heapster": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/heapster/influx-grafana-rc.yaml", constants.AddonsPath, "influxGrafana-rc.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/heapster/grafana-svc.yaml", constants.AddonsPath, "grafana-svc.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/heapster/influxdb-svc.yaml", constants.AddonsPath, "influxdb-svc.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/heapster/heapster-rc.yaml", constants.AddonsPath, "heapster-rc.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/heapster/heapster-svc.yaml", constants.AddonsPath, "heapster-svc.yaml", - "0640"), + "0640", + false), }, false, "heapster"), "efk": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/efk/elasticsearch-rc.yaml", constants.AddonsPath, "elasticsearch-rc.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/efk/elasticsearch-svc.yaml", constants.AddonsPath, "elasticsearch-svc.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/efk/fluentd-es-rc.yaml", constants.AddonsPath, "fluentd-es-rc.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/efk/fluentd-es-configmap.yaml", constants.AddonsPath, "fluentd-es-configmap.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/efk/kibana-rc.yaml", constants.AddonsPath, "kibana-rc.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/efk/kibana-svc.yaml", constants.AddonsPath, "kibana-svc.yaml", - "0640"), + "0640", + false), }, false, "efk"), "ingress": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/ingress/ingress-configmap.yaml", constants.AddonsPath, "ingress-configmap.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/ingress/ingress-rbac.yaml", constants.AddonsPath, "ingress-rbac.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/ingress/ingress-dp.yaml", constants.AddonsPath, "ingress-dp.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/ingress/ingress-svc.yaml", constants.AddonsPath, "ingress-svc.yaml", - "0640"), + "0640", + false), }, false, "ingress"), "metrics-server": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/metrics-server/metrics-apiservice.yaml", constants.AddonsPath, "metrics-apiservice.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/metrics-server/metrics-server-deployment.yaml", constants.AddonsPath, "metrics-server-deployment.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/metrics-server/metrics-server-service.yaml", constants.AddonsPath, "metrics-server-service.yaml", - "0640"), + "0640", + false), }, false, "metrics-server"), "registry": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/registry/registry-rc.yaml", constants.AddonsPath, "registry-rc.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/registry/registry-svc.yaml", constants.AddonsPath, "registry-svc.yaml", - "0640"), + "0640", + false), }, false, "registry"), "registry-creds": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/registry-creds/registry-creds-rc.yaml", constants.AddonsPath, "registry-creds-rc.yaml", - "0640"), + "0640", + false), }, false, "registry-creds"), "freshpod": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/freshpod/freshpod-rc.yaml", constants.AddonsPath, "freshpod-rc.yaml", - "0640"), + "0640", + false), }, false, "freshpod"), "nvidia-driver-installer": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/gpu/nvidia-driver-installer.yaml", constants.AddonsPath, "nvidia-driver-installer.yaml", - "0640"), + "0640", + false), }, false, "nvidia-driver-installer"), "nvidia-gpu-device-plugin": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/gpu/nvidia-gpu-device-plugin.yaml", constants.AddonsPath, "nvidia-gpu-device-plugin.yaml", - "0640"), + "0640", + false), }, false, "nvidia-gpu-device-plugin"), "logviewer": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/logviewer/logviewer-dp-and-svc.yaml", constants.AddonsPath, "logviewer-dp-and-svc.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/logviewer/logviewer-rbac.yaml", constants.AddonsPath, "logviewer-rbac.yaml", - "0640"), + "0640", + false), }, false, "logviewer"), "gvisor": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/gvisor/gvisor-pod.yaml", constants.AddonsPath, "gvisor-pod.yaml", - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/gvisor/gvisor-config.toml", constants.GvisorFilesPath, constants.GvisorConfigTomlTargetName, - "0640"), + "0640", + false), NewBinDataAsset( "deploy/addons/gvisor/gvisor-containerd-shim.toml", constants.GvisorFilesPath, constants.GvisorContainerdShimTargetName, - "0640"), + "0640", + false), }, false, "gvisor"), } diff --git a/pkg/minikube/assets/vm_assets.go b/pkg/minikube/assets/vm_assets.go index 27d668a7ea5e..8c5d5a463b9e 100644 --- a/pkg/minikube/assets/vm_assets.go +++ b/pkg/minikube/assets/vm_assets.go @@ -18,6 +18,7 @@ package assets import ( "bytes" + "html/template" "io" "os" "path" @@ -134,32 +135,73 @@ func NewMemoryAsset(d []byte, targetDir, targetName, permissions string) *Memory type BinDataAsset struct { BaseAsset + template *template.Template } -func NewBinDataAsset(assetName, targetDir, targetName, permissions string) *BinDataAsset { +func NewBinDataAsset(assetName, targetDir, targetName, permissions string, isTemplate bool) *BinDataAsset { m := &BinDataAsset{ - BaseAsset{ + BaseAsset: BaseAsset{ AssetName: assetName, TargetDir: targetDir, TargetName: targetName, Permissions: permissions, }, + template: nil, } - m.loadData() + m.loadData(isTemplate) return m } -func (m *BinDataAsset) loadData() error { +func defaultValue(defValue string, val interface{}) string { + if val == nil { + return defValue + } + strVal, ok := val.(string) + if !ok || strVal == "" { + return defValue + } + return strVal +} + +func (m *BinDataAsset) loadData(isTemplate bool) error { contents, err := Asset(m.AssetName) if err != nil { return err } + + if isTemplate { + tpl, err := template.New(m.AssetName).Funcs(template.FuncMap{"default": defaultValue}).Parse(string(contents)) + if err != nil { + return err + } + + m.template = tpl + } + m.data = contents m.Length = len(contents) m.reader = bytes.NewReader(m.data) return nil } +func (m *BinDataAsset) IsTemplate() bool { + return m.template != nil +} + +func (m *BinDataAsset) Evaluate(data interface{}) (*MemoryAsset, error){ + if !m.IsTemplate() { + return nil, errors.Errorf("the asset %s is not a template", m.AssetName) + + } + + var buf bytes.Buffer + if err := m.template.Execute(&buf, data); err != nil { + return nil, err + } + + return NewMemoryAsset(buf.Bytes(), m.GetTargetDir(), m.GetTargetName(), m.GetPermissions()), nil +} + func (m *BinDataAsset) GetLength() int { return m.Length } diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index 4140f58548a9..ecc7197f2bc3 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -50,10 +50,11 @@ const ( BootstrapperTypeKubeadm = "kubeadm" ) -func GetCachedImageList(version string, bootstrapper string) []string { +func GetCachedImageList(imageRepository string, version string, bootstrapper string) []string { switch bootstrapper { case BootstrapperTypeKubeadm: - return constants.GetKubeadmCachedImages(version) + _, images := constants.GetKubeadmImages(imageRepository, version) + return images default: return []string{} } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index aa68d68c0ffa..182f51ae7f78 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -32,7 +32,7 @@ import ( "github.com/docker/machine/libmachine" "github.com/docker/machine/libmachine/state" "github.com/golang/glog" - download "github.com/jimmidyson/go-download" + "github.com/jimmidyson/go-download" "github.com/pkg/errors" "golang.org/x/sync/errgroup" "k8s.io/minikube/pkg/minikube/assets" @@ -184,7 +184,7 @@ func (k *KubeadmBootstrapper) StartCluster(k8s config.KubernetesConfig) error { return nil } -func addAddons(files *[]assets.CopyableFile) error { +func addAddons(files *[]assets.CopyableFile, data interface{}) error { // add addons to file list // custom addons if err := assets.AddMinikubeDirAssets(files); err != nil { @@ -194,7 +194,16 @@ func addAddons(files *[]assets.CopyableFile) error { for _, addonBundle := range assets.Addons { if isEnabled, err := addonBundle.IsEnabled(); err == nil && isEnabled { for _, addon := range addonBundle.Assets { - *files = append(*files, addon) + if addon.IsTemplate() { + addonFile, err := addon.Evaluate(data) + if err != nil { + return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName()) + } + + *files = append(*files, addonFile) + } else { + *files = append(*files, addon) + } } } else if err != nil { return nil @@ -286,7 +295,10 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, extraOpts["network-plugin"] = k8s.NetworkPlugin } - extraFlags := convertToFlags(extraOpts) + podInfraContainerImage, _ := constants.GetKubeadmImages(k8s.ImageRepository, k8s.KubernetesVersion) + if _, ok := extraOpts["pod-infra-container-image"]; !ok && k8s.ImageRepository != "" && podInfraContainerImage != "" { + extraOpts["pod-infra-container-image"] = podInfraContainerImage + } // parses a map of the feature gates for kubelet _, kubeletFeatureArgs, err := ParseFeatureArgs(k8s.FeatureGates) @@ -294,14 +306,18 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, return "", errors.Wrap(err, "parses feature gate config for kubelet") } + if kubeletFeatureArgs != "" { + extraOpts["feature-gates"] = kubeletFeatureArgs + } + + extraFlags := convertToFlags(extraOpts) + b := bytes.Buffer{} opts := struct { ExtraOptions string - FeatureGates string ContainerRuntime string }{ ExtraOptions: extraFlags, - FeatureGates: kubeletFeatureArgs, ContainerRuntime: k8s.ContainerRuntime, } if err := kubeletSystemdTemplate.Execute(&b, opts); err != nil { @@ -312,8 +328,9 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, } func (k *KubeadmBootstrapper) UpdateCluster(cfg config.KubernetesConfig) error { + _, images := constants.GetKubeadmImages(cfg.ImageRepository, cfg.KubernetesVersion) if cfg.ShouldLoadCachedImages { - err := machine.LoadImages(k.c, constants.GetKubeadmCachedImages(cfg.KubernetesVersion), constants.ImageCacheDir) + err := machine.LoadImages(k.c, images, constants.ImageCacheDir) if err != nil { return errors.Wrap(err, "loading cached images") } @@ -322,7 +339,7 @@ func (k *KubeadmBootstrapper) UpdateCluster(cfg config.KubernetesConfig) error { if err != nil { return errors.Wrap(err, "runtime") } - kubeadmCfg, err := generateConfig(cfg, r) + kubeadmCfg, opts, err := generateConfig(cfg, r) if err != nil { return errors.Wrap(err, "generating kubeadm cfg") } @@ -370,7 +387,7 @@ func (k *KubeadmBootstrapper) UpdateCluster(cfg config.KubernetesConfig) error { return errors.Wrap(err, "downloading binaries") } - if err := addAddons(&files); err != nil { + if err := addAddons(&files, opts); err != nil { return errors.Wrap(err, "adding addons") } @@ -391,22 +408,22 @@ sudo systemctl start kubelet return nil } -func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, error) { +func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, interface{}, error) { version, err := ParseKubernetesVersion(k8s.KubernetesVersion) if err != nil { - return "", errors.Wrap(err, "parsing kubernetes version") + return "", nil, errors.Wrap(err, "parsing kubernetes version") } // parses a map of the feature gates for kubeadm and component kubeadmFeatureArgs, componentFeatureArgs, err := ParseFeatureArgs(k8s.FeatureGates) if err != nil { - return "", errors.Wrap(err, "parses feature gate config for kubeadm and component") + return "", nil, errors.Wrap(err, "parses feature gate config for kubeadm and component") } // generates a map of component to extra args for apiserver, controller-manager, and scheduler extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version, componentFeatureArgs) if err != nil { - return "", errors.Wrap(err, "generating extra component config for kubeadm") + return "", nil, errors.Wrap(err, "generating extra component config for kubeadm") } // In case of no port assigned, use util.APIServerPort @@ -424,6 +441,7 @@ func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, er EtcdDataDir string NodeName string CRISocket string + ImageRepository string ExtraArgs []ComponentExtraArgs FeatureArgs map[string]bool NoTaintMaster bool @@ -436,6 +454,7 @@ func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, er EtcdDataDir: "/data/minikube", //TODO(r2d4): change to something else persisted NodeName: k8s.NodeName, CRISocket: r.SocketPath(), + ImageRepository: k8s.ImageRepository, ExtraArgs: extraComponentConfig, FeatureArgs: kubeadmFeatureArgs, NoTaintMaster: false, // That does not work with k8s 1.12+ @@ -455,10 +474,10 @@ func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, er kubeadmConfigTemplate = kubeadmConfigTemplateV1Alpha3 } if err := kubeadmConfigTemplate.Execute(&b, opts); err != nil { - return "", err + return "", nil, err } - return b.String(), nil + return b.String(), opts, nil } func maybeDownloadAndCache(binary, version string) (string, error) { diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go index 01ad19cd03dc..10e780de9c13 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go @@ -264,7 +264,7 @@ apiServerExtraArgs: } t.Run(test.description, func(t *testing.T) { - actualCfg, err := generateConfig(test.cfg, runtime) + actualCfg, _, err := generateConfig(test.cfg, runtime) if err != nil && !test.shouldErr { t.Errorf("got unexpected error generating config: %v", err) return diff --git a/pkg/minikube/bootstrapper/kubeadm/templates.go b/pkg/minikube/bootstrapper/kubeadm/templates.go index 8b35b2b827b5..ee27272fe3f2 100644 --- a/pkg/minikube/bootstrapper/kubeadm/templates.go +++ b/pkg/minikube/bootstrapper/kubeadm/templates.go @@ -38,7 +38,8 @@ networking: etcd: dataDir: {{.EtcdDataDir}} nodeName: {{.NodeName}} -{{if .CRISocket}}criSocket: {{.CRISocket}} +{{if .ImageRepository}}imageRepository: {{.ImageRepository}} +{{end}}{{if .CRISocket}}criSocket: {{.CRISocket}} {{end}}{{range .ExtraArgs}}{{.Component}}:{{range $i, $val := printMapInOrder .Options ": " }} {{$val}}{{end}} {{end}}{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}} @@ -67,7 +68,8 @@ nodeRegistration: --- apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration -{{range .ExtraArgs}}{{.Component}}:{{range $i, $val := printMapInOrder .Options ": " }} +{{if .ImageRepository}}imageRepository: {{.ImageRepository}} +{{end}}{{range .ExtraArgs}}{{.Component}}:{{range $i, $val := printMapInOrder .Options ": " }} {{$val}}{{end}} {{end}}{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}} {{$i}}: {{$val}}{{end}} @@ -101,7 +103,7 @@ var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate" [Service] ExecStart= -ExecStart=/usr/bin/kubelet {{.ExtraOptions}} {{if .FeatureGates}}--feature-gates={{.FeatureGates}}{{end}} +ExecStart=/usr/bin/kubelet{{if .ExtraOptions}} {{.ExtraOptions}}{{end}} [Install] `)) diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 3f9a7a1c3ed7..7982d04c4cb4 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -69,6 +69,7 @@ type KubernetesConfig struct { NetworkPlugin string FeatureGates string ServiceCIDR string + ImageRepository string ExtraOptions util.ExtraOptionSlice ShouldLoadCachedImages bool diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 2fa11c50c7d3..8182cca43e1d 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -176,13 +176,24 @@ const IsMinikubeChildProcess = "IS_MINIKUBE_CHILD_PROCESS" const DriverNone = "none" const FileScheme = "file" -func GetKubeadmCachedImages(kubernetesVersionStr string) []string { +func GetKubeadmImages(imageRepository string, kubernetesVersionStr string) (string, []string) { + minikubeRepository := imageRepository + if imageRepository == "" { + imageRepository = "k8s.gcr.io" + minikubeRepository = "gcr.io/k8s-minikube" + } + if !strings.HasSuffix(imageRepository, "/") { + imageRepository += "/" + } + if !strings.HasSuffix(minikubeRepository, "/") { + minikubeRepository += "/" + } var images = []string{ - "k8s.gcr.io/kube-proxy-amd64:" + kubernetesVersionStr, - "k8s.gcr.io/kube-scheduler-amd64:" + kubernetesVersionStr, - "k8s.gcr.io/kube-controller-manager-amd64:" + kubernetesVersionStr, - "k8s.gcr.io/kube-apiserver-amd64:" + kubernetesVersionStr, + imageRepository + "kube-proxy-amd64:" + kubernetesVersionStr, + imageRepository + "kube-scheduler-amd64:" + kubernetesVersionStr, + imageRepository + "kube-controller-manager-amd64:" + kubernetesVersionStr, + imageRepository + "kube-apiserver-amd64:" + kubernetesVersionStr, } ge_v1_13 := semver.MustParseRange(">=1.13.0") @@ -197,74 +208,84 @@ func GetKubeadmCachedImages(kubernetesVersionStr string) []string { glog.Errorln("Error parsing version semver: ", err) } + var podInfraContainerImage string if ge_v1_13(kubernetesVersion) { + podInfraContainerImage = imageRepository + "pause-amd64:3.1" images = append(images, []string{ - "k8s.gcr.io/pause-amd64:3.1", - "k8s.gcr.io/pause:3.1", - "k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.8", - "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.8", - "k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.8", - "k8s.gcr.io/etcd-amd64:3.2.24", - "k8s.gcr.io/coredns:1.2.6", + podInfraContainerImage, + imageRepository + "pause:3.1", + imageRepository + "k8s-dns-kube-dns-amd64:1.14.8", + imageRepository + "k8s-dns-dnsmasq-nanny-amd64:1.14.8", + imageRepository + "k8s-dns-sidecar-amd64:1.14.8", + imageRepository + "etcd-amd64:3.2.24", + imageRepository + "coredns:1.2.6", }...) } else if v1_12(kubernetesVersion) { + podInfraContainerImage = imageRepository + "pause-amd64:3.1" images = append(images, []string{ - "k8s.gcr.io/pause-amd64:3.1", - "k8s.gcr.io/pause:3.1", - "k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.8", - "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.8", - "k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.8", - "k8s.gcr.io/etcd-amd64:3.2.24", - "k8s.gcr.io/coredns:1.2.2", + podInfraContainerImage, + imageRepository + "pause:3.1", + imageRepository + "k8s-dns-kube-dns-amd64:1.14.8", + imageRepository + "k8s-dns-dnsmasq-nanny-amd64:1.14.8", + imageRepository + "k8s-dns-sidecar-amd64:1.14.8", + imageRepository + "etcd-amd64:3.2.24", + imageRepository + "coredns:1.2.2", }...) } else if v1_11(kubernetesVersion) { + podInfraContainerImage = imageRepository + "pause-amd64:3.1" images = append(images, []string{ - "k8s.gcr.io/pause-amd64:3.1", - "k8s.gcr.io/pause:3.1", - "k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.8", - "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.8", - "k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.8", - "k8s.gcr.io/etcd-amd64:3.2.18", - "k8s.gcr.io/coredns:1.1.3", + podInfraContainerImage, + imageRepository + "pause:3.1", + imageRepository + "k8s-dns-kube-dns-amd64:1.14.8", + imageRepository + "k8s-dns-dnsmasq-nanny-amd64:1.14.8", + imageRepository + "k8s-dns-sidecar-amd64:1.14.8", + imageRepository + "etcd-amd64:3.2.18", + imageRepository + "coredns:1.1.3", }...) } else if v1_10(kubernetesVersion) { + podInfraContainerImage = imageRepository + "pause-amd64:3.1" images = append(images, []string{ - "k8s.gcr.io/pause-amd64:3.1", - "k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.8", - "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.8", - "k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.8", - "k8s.gcr.io/etcd-amd64:3.1.12", + podInfraContainerImage, + imageRepository + "k8s-dns-kube-dns-amd64:1.14.8", + imageRepository + "k8s-dns-dnsmasq-nanny-amd64:1.14.8", + imageRepository + "k8s-dns-sidecar-amd64:1.14.8", + imageRepository + "etcd-amd64:3.1.12", }...) } else if v1_9(kubernetesVersion) { + podInfraContainerImage = imageRepository + "pause-amd64:3.0" images = append(images, []string{ - "k8s.gcr.io/pause-amd64:3.0", - "k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.7", - "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.7", - "k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.7", - "k8s.gcr.io/etcd-amd64:3.1.10", + podInfraContainerImage, + imageRepository + "k8s-dns-kube-dns-amd64:1.14.7", + imageRepository + "k8s-dns-dnsmasq-nanny-amd64:1.14.7", + imageRepository + "k8s-dns-sidecar-amd64:1.14.7", + imageRepository + "etcd-amd64:3.1.10", }...) } else if v1_8(kubernetesVersion) { + podInfraContainerImage = imageRepository + "pause-amd64:3.0" images = append(images, []string{ - "k8s.gcr.io/pause-amd64:3.0", - "k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.5", - "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.5", - "k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.5", - "k8s.gcr.io/etcd-amd64:3.0.17", + podInfraContainerImage, + imageRepository + "k8s-dns-kube-dns-amd64:1.14.5", + imageRepository + "k8s-dns-dnsmasq-nanny-amd64:1.14.5", + imageRepository + "k8s-dns-sidecar-amd64:1.14.5", + imageRepository + "etcd-amd64:3.0.17", }...) + + } else { + podInfraContainerImage = imageRepository + "/pause-amd64:3.0" } images = append(images, []string{ - "k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1", - "k8s.gcr.io/kube-addon-manager:v8.6", - "gcr.io/k8s-minikube/storage-provisioner:v1.8.1", + imageRepository + "kubernetes-dashboard-amd64:v1.10.1", + imageRepository + "kube-addon-manager:v8.6", + minikubeRepository + "storage-provisioner:v1.8.1", }...) - return images + return podInfraContainerImage, images } var ImageCacheDir = MakeMiniPath("cache", "images") diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index 9123b62c9122..2afcc6052218 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -48,8 +48,8 @@ var getWindowsVolumeName = getWindowsVolumeNameCmd // loadImageLock is used to serialize image loads to avoid overloading the guest VM var loadImageLock sync.Mutex -func CacheImagesForBootstrapper(version string, clusterBootstrapper string) error { - images := bootstrapper.GetCachedImageList(version, clusterBootstrapper) +func CacheImagesForBootstrapper(imageRepository string, version string, clusterBootstrapper string) error { + images := bootstrapper.GetCachedImageList(imageRepository, version, clusterBootstrapper) if err := CacheImages(images, constants.ImageCacheDir); err != nil { return errors.Wrapf(err, "Caching images for %s", clusterBootstrapper) From 7a59d376cfa8beaa10f6c95c10586e60ad167860 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Mon, 18 Feb 2019 01:00:18 +0800 Subject: [PATCH 2/6] Turn addon resource with images into template When the user overrides image repository the images will be pulled from the overrided one instead of the official repositories. --- deploy/addons/efk/elasticsearch-rc.yaml | 2 +- deploy/addons/efk/fluentd-es-rc.yaml | 2 +- deploy/addons/freshpod/freshpod-rc.yaml | 2 +- .../addons/gpu/nvidia-driver-installer.yaml | 4 ++-- .../addons/gpu/nvidia-gpu-device-plugin.yaml | 2 +- deploy/addons/gvisor/gvisor-config.toml | 2 +- deploy/addons/gvisor/gvisor-pod.yaml | 2 +- deploy/addons/heapster/heapster-rc.yaml | 2 +- deploy/addons/heapster/influx-grafana-rc.yaml | 4 ++-- deploy/addons/ingress/ingress-dp.yaml | 2 +- .../metrics-server-deployment.yaml | 2 +- pkg/minikube/assets/addons.go | 22 +++++++++---------- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/deploy/addons/efk/elasticsearch-rc.yaml b/deploy/addons/efk/elasticsearch-rc.yaml index 525350bb36e4..54dad369bbcc 100644 --- a/deploy/addons/efk/elasticsearch-rc.yaml +++ b/deploy/addons/efk/elasticsearch-rc.yaml @@ -34,7 +34,7 @@ spec: spec: containers: - name: elasticsearch-logging - image: k8s.gcr.io/elasticsearch:v5.6.2 + image: {{default "k8s.gcr.io" .ImageRepository}}/elasticsearch:v5.6.2 resources: limits: cpu: 500m diff --git a/deploy/addons/efk/fluentd-es-rc.yaml b/deploy/addons/efk/fluentd-es-rc.yaml index 0c5e3581ebf5..75e38698417c 100644 --- a/deploy/addons/efk/fluentd-es-rc.yaml +++ b/deploy/addons/efk/fluentd-es-rc.yaml @@ -31,7 +31,7 @@ spec: spec: containers: - name: fluentd-es - image: k8s.gcr.io/fluentd-elasticsearch:v2.0.2 + image: {{default "k8s.gcr.io" .ImageRepository}}/fluentd-elasticsearch:v2.0.2 env: - name: FLUENTD_ARGS value: --no-supervisor -q diff --git a/deploy/addons/freshpod/freshpod-rc.yaml b/deploy/addons/freshpod/freshpod-rc.yaml index 71680c7cf139..6d2b9ba5b764 100644 --- a/deploy/addons/freshpod/freshpod-rc.yaml +++ b/deploy/addons/freshpod/freshpod-rc.yaml @@ -34,7 +34,7 @@ spec: spec: containers: - name: freshpod - image: gcr.io/google-samples/freshpod:v0.0.1 + image: {{default "gcr.io/google-samples" .ImageRepository}}/freshpod:v0.0.1 imagePullPolicy: IfNotPresent volumeMounts: - name: docker diff --git a/deploy/addons/gpu/nvidia-driver-installer.yaml b/deploy/addons/gpu/nvidia-driver-installer.yaml index e5a03efc2017..c6e5a106e62a 100644 --- a/deploy/addons/gpu/nvidia-driver-installer.yaml +++ b/deploy/addons/gpu/nvidia-driver-installer.yaml @@ -50,7 +50,7 @@ spec: hostPath: path: / initContainers: - - image: k8s.gcr.io/minikube-nvidia-driver-installer@sha256:492d46f2bc768d6610ec5940b6c3c33c75e03e201cc8786e04cc488659fd6342 + - image: {{default "k8s.gcr.io" .ImageRepository}}/minikube-nvidia-driver-installer@sha256:492d46f2bc768d6610ec5940b6c3c33c75e03e201cc8786e04cc488659fd6342 name: nvidia-driver-installer resources: requests: @@ -72,5 +72,5 @@ spec: - name: root-mount mountPath: /root containers: - - image: "gcr.io/google-containers/pause:2.0" + - image: "{{default "gcr.io/google-containers" .ImageRepository}}/pause:2.0" name: pause diff --git a/deploy/addons/gpu/nvidia-gpu-device-plugin.yaml b/deploy/addons/gpu/nvidia-gpu-device-plugin.yaml index 8d403735bc4e..96252d1ba5fd 100644 --- a/deploy/addons/gpu/nvidia-gpu-device-plugin.yaml +++ b/deploy/addons/gpu/nvidia-gpu-device-plugin.yaml @@ -46,7 +46,7 @@ spec: hostPath: path: /dev containers: - - image: "k8s.gcr.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e" + - image: "{{default "k8s.gcr.io" .ImageRepository}}/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e" command: ["/usr/bin/nvidia-gpu-device-plugin", "-logtostderr"] name: nvidia-gpu-device-plugin resources: diff --git a/deploy/addons/gvisor/gvisor-config.toml b/deploy/addons/gvisor/gvisor-config.toml index c891cddbee6d..034be8a4e57c 100644 --- a/deploy/addons/gvisor/gvisor-config.toml +++ b/deploy/addons/gvisor/gvisor-config.toml @@ -29,7 +29,7 @@ oom_score = 0 stream_server_address = "" stream_server_port = "10010" enable_selinux = false - sandbox_image = "k8s.gcr.io/pause:3.1" + sandbox_image = "{{default "k8s.gcr.io" .ImageRepository}}/pause:3.1" stats_collect_period = 10 systemd_cgroup = false enable_tls_streaming = false diff --git a/deploy/addons/gvisor/gvisor-pod.yaml b/deploy/addons/gvisor/gvisor-pod.yaml index 8cb0d3321977..08ecd7225f53 100644 --- a/deploy/addons/gvisor/gvisor-pod.yaml +++ b/deploy/addons/gvisor/gvisor-pod.yaml @@ -24,7 +24,7 @@ spec: hostPID: true containers: - name: gvisor - image: gcr.io/k8s-minikube/gvisor-addon:latest + image: {{default "gcr.io/k8s-minikube" .ImageRepository}}/gvisor-addon:latest securityContext: privileged: true volumeMounts: diff --git a/deploy/addons/heapster/heapster-rc.yaml b/deploy/addons/heapster/heapster-rc.yaml index db49b165a43b..d9c2cc36fcd9 100644 --- a/deploy/addons/heapster/heapster-rc.yaml +++ b/deploy/addons/heapster/heapster-rc.yaml @@ -37,7 +37,7 @@ spec: spec: containers: - name: heapster - image: k8s.gcr.io/heapster-amd64:v1.5.3 + image: {{default "k8s.gcr.io" .ImageRepository}}/heapster-amd64:v1.5.3 imagePullPolicy: IfNotPresent command: - /heapster diff --git a/deploy/addons/heapster/influx-grafana-rc.yaml b/deploy/addons/heapster/influx-grafana-rc.yaml index 328d4bdd1d0d..2e988c34ae2b 100644 --- a/deploy/addons/heapster/influx-grafana-rc.yaml +++ b/deploy/addons/heapster/influx-grafana-rc.yaml @@ -34,7 +34,7 @@ spec: spec: containers: - name: influxdb - image: k8s.gcr.io/heapster-influxdb-amd64:v1.3.3 + image: {{default "k8s.gcr.io" .ImageRepository}}/heapster-influxdb-amd64:v1.3.3 imagePullPolicy: IfNotPresent ports: - name: http @@ -45,7 +45,7 @@ spec: - mountPath: /data name: influxdb-storage - name: grafana - image: k8s.gcr.io/heapster-grafana-amd64:v4.4.3 + image: {{default "k8s.gcr.io" .ImageRepository}}/heapster-grafana-amd64:v4.4.3 imagePullPolicy: IfNotPresent env: - name: INFLUXDB_SERVICE_URL diff --git a/deploy/addons/ingress/ingress-dp.yaml b/deploy/addons/ingress/ingress-dp.yaml index 7d0c885a79cf..510a5deed014 100644 --- a/deploy/addons/ingress/ingress-dp.yaml +++ b/deploy/addons/ingress/ingress-dp.yaml @@ -39,7 +39,7 @@ spec: # Any image is permissible as long as: # 1. It serves a 404 page at / # 2. It serves 200 on a /healthz endpoint - image: gcr.io/google_containers/defaultbackend:1.4 + image: {{default "gcr.io/google_containers" .ImageRepository}}/defaultbackend:1.4 imagePullPolicy: IfNotPresent livenessProbe: httpGet: diff --git a/deploy/addons/metrics-server/metrics-server-deployment.yaml b/deploy/addons/metrics-server/metrics-server-deployment.yaml index c2a86b5ac79b..34350a296318 100644 --- a/deploy/addons/metrics-server/metrics-server-deployment.yaml +++ b/deploy/addons/metrics-server/metrics-server-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: metrics-server - image: k8s.gcr.io/metrics-server-amd64:v0.2.1 + image: {{default "k8s.gcr.io" .ImageRepository}}/metrics-server-amd64:v0.2.1 imagePullPolicy: Always command: - /metrics-server diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index e21a5aba1f58..70420683398d 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -127,7 +127,7 @@ var Addons = map[string]*Addon{ constants.AddonsPath, "influxGrafana-rc.yaml", "0640", - false), + true), NewBinDataAsset( "deploy/addons/heapster/grafana-svc.yaml", constants.AddonsPath, @@ -145,7 +145,7 @@ var Addons = map[string]*Addon{ constants.AddonsPath, "heapster-rc.yaml", "0640", - false), + true), NewBinDataAsset( "deploy/addons/heapster/heapster-svc.yaml", constants.AddonsPath, @@ -159,7 +159,7 @@ var Addons = map[string]*Addon{ constants.AddonsPath, "elasticsearch-rc.yaml", "0640", - false), + true), NewBinDataAsset( "deploy/addons/efk/elasticsearch-svc.yaml", constants.AddonsPath, @@ -171,7 +171,7 @@ var Addons = map[string]*Addon{ constants.AddonsPath, "fluentd-es-rc.yaml", "0640", - false), + true), NewBinDataAsset( "deploy/addons/efk/fluentd-es-configmap.yaml", constants.AddonsPath, @@ -209,7 +209,7 @@ var Addons = map[string]*Addon{ constants.AddonsPath, "ingress-dp.yaml", "0640", - false), + true), NewBinDataAsset( "deploy/addons/ingress/ingress-svc.yaml", constants.AddonsPath, @@ -229,7 +229,7 @@ var Addons = map[string]*Addon{ constants.AddonsPath, "metrics-server-deployment.yaml", "0640", - false), + true), NewBinDataAsset( "deploy/addons/metrics-server/metrics-server-service.yaml", constants.AddonsPath, @@ -265,7 +265,7 @@ var Addons = map[string]*Addon{ constants.AddonsPath, "freshpod-rc.yaml", "0640", - false), + true), }, false, "freshpod"), "nvidia-driver-installer": NewAddon([]*BinDataAsset{ NewBinDataAsset( @@ -273,7 +273,7 @@ var Addons = map[string]*Addon{ constants.AddonsPath, "nvidia-driver-installer.yaml", "0640", - false), + true), }, false, "nvidia-driver-installer"), "nvidia-gpu-device-plugin": NewAddon([]*BinDataAsset{ NewBinDataAsset( @@ -281,7 +281,7 @@ var Addons = map[string]*Addon{ constants.AddonsPath, "nvidia-gpu-device-plugin.yaml", "0640", - false), + true), }, false, "nvidia-gpu-device-plugin"), "logviewer": NewAddon([]*BinDataAsset{ NewBinDataAsset( @@ -303,13 +303,13 @@ var Addons = map[string]*Addon{ constants.AddonsPath, "gvisor-pod.yaml", "0640", - false), + true), NewBinDataAsset( "deploy/addons/gvisor/gvisor-config.toml", constants.GvisorFilesPath, constants.GvisorConfigTomlTargetName, "0640", - false), + true), NewBinDataAsset( "deploy/addons/gvisor/gvisor-containerd-shim.toml", constants.GvisorFilesPath, From 39fcf1fe209495ac6d7bcfbbf5c383059f418ec7 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Mon, 18 Feb 2019 01:00:22 +0800 Subject: [PATCH 3/6] Add test cases for custom image repository feature --- .../bootstrapper/kubeadm/kubeadm_test.go | 107 +++++++++++++++++- 1 file changed, 102 insertions(+), 5 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go index 10e780de9c13..f2aa743380b8 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go @@ -19,6 +19,7 @@ package kubeadm import ( "testing" + "github.com/google/go-cmp/cmp" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/util" @@ -26,10 +27,11 @@ import ( func TestGenerateConfig(t *testing.T) { tests := []struct { - description string - cfg config.KubernetesConfig - expectedCfg string - shouldErr bool + description string + cfg config.KubernetesConfig + expectedCfg string + expectedOpts interface{} + shouldErr bool }{ { description: "no extra args", @@ -55,6 +57,37 @@ nodeName: minikube apiServerExtraArgs: admission-control: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" `, + expectedOpts: struct { + CertDir string + ServiceCIDR string + AdvertiseAddress string + APIServerPort int + KubernetesVersion string + EtcdDataDir string + NodeName string + CRISocket string + ImageRepository string + ExtraArgs []ComponentExtraArgs + FeatureArgs map[string]bool + NoTaintMaster bool + }{ + CertDir: util.DefaultCertPath, + ServiceCIDR: util.DefaultServiceCIDR, + AdvertiseAddress: "192.168.1.100", + APIServerPort: 8443, + KubernetesVersion: "v1.10.0", + EtcdDataDir: "/data/minikube", + NodeName: "minikube", + ExtraArgs: []ComponentExtraArgs{ + { + Component: "apiServerExtraArgs", + Options: map[string]string{ + "admission-control": "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", + }, + }, + }, + FeatureArgs: map[string]bool{}, + NoTaintMaster: true}, }, { description: "extra args all components", @@ -255,6 +288,65 @@ apiServerExtraArgs: admission-control: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" `, }, + { + description: "custom image repository", + cfg: config.KubernetesConfig{ + NodeIP: "192.168.1.100", + KubernetesVersion: "v1.10.0", + NodeName: "minikube", + ImageRepository: "docker-proxy-image.io/google_containers", + }, + expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1 +kind: MasterConfiguration +noTaintMaster: true +api: + advertiseAddress: 192.168.1.100 + bindPort: 8443 + controlPlaneEndpoint: localhost +kubernetesVersion: v1.10.0 +certificatesDir: /var/lib/minikube/certs/ +networking: + serviceSubnet: 10.96.0.0/12 +etcd: + dataDir: /data/minikube +nodeName: minikube +imageRepository: docker-proxy-image.io/google_containers +apiServerExtraArgs: + admission-control: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" +`, + expectedOpts: struct { + CertDir string + ServiceCIDR string + AdvertiseAddress string + APIServerPort int + KubernetesVersion string + EtcdDataDir string + NodeName string + CRISocket string + ImageRepository string + ExtraArgs []ComponentExtraArgs + FeatureArgs map[string]bool + NoTaintMaster bool + }{ + CertDir: util.DefaultCertPath, + ServiceCIDR: util.DefaultServiceCIDR, + AdvertiseAddress: "192.168.1.100", + APIServerPort: 8443, + KubernetesVersion: "v1.10.0", + EtcdDataDir: "/data/minikube", + NodeName: "minikube", + ExtraArgs: []ComponentExtraArgs{ + { + Component: "apiServerExtraArgs", + Options: map[string]string{ + "admission-control": "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", + }, + }, + }, + ImageRepository: "docker-proxy-image.io/google_containers", + FeatureArgs: map[string]bool{}, + NoTaintMaster: true}, + }, } for _, test := range tests { @@ -264,7 +356,7 @@ apiServerExtraArgs: } t.Run(test.description, func(t *testing.T) { - actualCfg, _, err := generateConfig(test.cfg, runtime) + actualCfg, actualOpts, err := generateConfig(test.cfg, runtime) if err != nil && !test.shouldErr { t.Errorf("got unexpected error generating config: %v", err) return @@ -277,6 +369,11 @@ apiServerExtraArgs: t.Errorf("actual config does not match expected. actual:\n%sexpected:\n%s", actualCfg, test.expectedCfg) return } + if test.expectedOpts != nil { + if diff := cmp.Diff(test.expectedOpts, actualOpts); diff != "" { + t.Errorf("opts differ: (-want +got)\n%s", diff) + } + } }) } } From e753a90be0a6a0b5414c1b723ff0f1eeacfe8f94 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Mon, 18 Feb 2019 01:00:27 +0800 Subject: [PATCH 4/6] Add test cases for NewKubeletConfig --- .../bootstrapper/kubeadm/kubeadm_test.go | 90 +++++++++++++++++++ pkg/minikube/bootstrapper/kubeadm/versions.go | 9 +- 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go index f2aa743380b8..cc61f0197985 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go @@ -25,6 +25,96 @@ import ( "k8s.io/minikube/pkg/util" ) +func TestGenerateKubeletConfig(t *testing.T) { + tests := []struct { + description string + cfg config.KubernetesConfig + expectedCfg string + shouldErr bool + }{ + { + description: "docker runtime", + cfg: config.KubernetesConfig{ + NodeIP: "192.168.1.100", + KubernetesVersion: "v1.1.0", + NodeName: "minikube", + ContainerRuntime: "docker", + }, + expectedCfg: ` +[Unit] +Wants=docker.socket + +[Service] +ExecStart= +ExecStart=/usr/bin/kubelet --allow-privileged=true --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cadvisor-port=0 --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=docker --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests --require-kubeconfig=true + +[Install] +`, + }, + { + description: "cri runtime", + cfg: config.KubernetesConfig{ + NodeIP: "192.168.1.100", + KubernetesVersion: "v1.1.0", + NodeName: "minikube", + ContainerRuntime: "cri-o", + }, + expectedCfg: ` +[Unit] +Wants=crio.service + +[Service] +ExecStart= +ExecStart=/usr/bin/kubelet --allow-privileged=true --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cadvisor-port=0 --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=/var/run/crio/crio.sock --hostname-override=minikube --image-service-endpoint=/var/run/crio/crio.sock --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests --require-kubeconfig=true --runtime-request-timeout=15m + +[Install] +`, + }, + { + description: "docker runtime with custom image repository", + cfg: config.KubernetesConfig{ + NodeIP: "192.168.1.100", + KubernetesVersion: "v1.1.0", + NodeName: "minikube", + ContainerRuntime: "docker", + ImageRepository: "docker-proxy-image.io/google_containers", + }, + expectedCfg: ` +[Unit] +Wants=docker.socket + +[Service] +ExecStart= +ExecStart=/usr/bin/kubelet --allow-privileged=true --authorization-mode=Webhook --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cadvisor-port=0 --cgroup-driver=cgroupfs --client-ca-file=/var/lib/minikube/certs/ca.crt --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --container-runtime=docker --hostname-override=minikube --kubeconfig=/etc/kubernetes/kubelet.conf --pod-infra-container-image=docker-proxy-image.io/google_containers//pause-amd64:3.0 --pod-manifest-path=/etc/kubernetes/manifests --require-kubeconfig=true + +[Install] +`, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + runtime, err := cruntime.New(cruntime.Config{Type: test.cfg.ContainerRuntime}) + if err != nil { + t.Fatalf("runtime: %v", err) + } + + actualCfg, err := NewKubeletConfig(test.cfg, runtime) + if err != nil && !test.shouldErr { + t.Errorf("got unexpected error generating config: %v", err) + return + } + if err == nil && test.shouldErr { + t.Errorf("expected error but got none, config: %s", actualCfg) + return + } + if diff := cmp.Diff(test.expectedCfg, actualCfg); diff != "" { + t.Errorf("actual config does not match expected. (-want +got)\n%s", diff) + } + }) + } +} + func TestGenerateConfig(t *testing.T) { tests := []struct { description string diff --git a/pkg/minikube/bootstrapper/kubeadm/versions.go b/pkg/minikube/bootstrapper/kubeadm/versions.go index fc2201cab2b6..ea7603143831 100644 --- a/pkg/minikube/bootstrapper/kubeadm/versions.go +++ b/pkg/minikube/bootstrapper/kubeadm/versions.go @@ -164,8 +164,13 @@ func ParseKubernetesVersion(version string) (semver.Version, error) { func convertToFlags(opts map[string]string) string { var flags []string - for k, v := range opts { - flags = append(flags, fmt.Sprintf("--%s=%s", k, v)) + var keys []string + for k := range opts { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + flags = append(flags, fmt.Sprintf("--%s=%s", k, opts[k])) } return strings.Join(flags, " ") } From ba5a02596f8ee392d275a96a34fdae82c60e8868 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Thu, 28 Feb 2019 10:15:08 +0800 Subject: [PATCH 5/6] Change --image-repository usage text --- cmd/minikube/cmd/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index a9be0d00d862..e054414519eb 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -124,7 +124,7 @@ func init() { startCmd.Flags().String(serviceCIDR, pkgutil.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.") startCmd.Flags().StringSliceVar(&insecureRegistry, "insecure-registry", nil, "Insecure Docker registries to pass to the Docker daemon. The default service CIDR range will automatically be added.") startCmd.Flags().StringSliceVar(®istryMirror, "registry-mirror", nil, "Registry mirrors to pass to the Docker daemon") - startCmd.Flags().String(imageRepository, "", "Image repository to pull down docker images.") + startCmd.Flags().String(imageRepository, "", "Alternative image repository to pull docker images from. This can be used when you have limited access to gcr.io. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers") startCmd.Flags().String(containerRuntime, "docker", "The container runtime to be used (docker, crio, containerd)") startCmd.Flags().String(criSocket, "", "The cri socket path to be used") startCmd.Flags().String(kubernetesVersion, constants.DefaultKubernetesVersion, "The kubernetes version that the minikube VM will use (ex: v1.2.3)") From daec030cdfb543e2314b0e13d7a55b0d5901de0a Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Fri, 22 Mar 2019 13:33:11 +0800 Subject: [PATCH 6/6] Render asset template when enabling/disabling addon --- cmd/minikube/cmd/config/util.go | 33 +++++++- pkg/minikube/assets/addons.go | 11 +++ pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 16 ++-- .../bootstrapper/kubeadm/kubeadm_test.go | 79 ++----------------- pkg/minikube/constants/constants.go | 2 +- test/integration/util/util.go | 1 + 6 files changed, 57 insertions(+), 85 deletions(-) diff --git a/cmd/minikube/cmd/config/util.go b/cmd/minikube/cmd/config/util.go index 586981df1e34..e5b0986c9e1a 100644 --- a/cmd/minikube/cmd/config/util.go +++ b/cmd/minikube/cmd/config/util.go @@ -18,6 +18,7 @@ package config import ( "fmt" + "os" "strconv" "strings" @@ -26,6 +27,7 @@ import ( "k8s.io/minikube/pkg/minikube/cluster" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/storageclass" ) @@ -123,15 +125,42 @@ func EnableOrDisableAddon(name string, val string) error { if err != nil { return errors.Wrap(err, "command runner") } + + cfg, err := config.Load() + if err != nil && !os.IsNotExist(err) { + exit.WithCode(exit.Data, "Unable to load config: %v", err) + } + + data := assets.GenerateTemplateData(cfg.KubernetesConfig) if enable { for _, addon := range addon.Assets { - if err := cmd.Copy(addon); err != nil { + var addonFile assets.CopyableFile + if addon.IsTemplate() { + addonFile, err = addon.Evaluate(data) + if err != nil { + return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName()) + } + + } else { + addonFile = addon + } + if err := cmd.Copy(addonFile); err != nil { return errors.Wrapf(err, "enabling addon %s", addon.AssetName) } } } else { for _, addon := range addon.Assets { - if err := cmd.Remove(addon); err != nil { + var addonFile assets.CopyableFile + if addon.IsTemplate() { + addonFile, err = addon.Evaluate(data) + if err != nil { + return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName()) + } + + } else { + addonFile = addon + } + if err := cmd.Remove(addonFile); err != nil { return errors.Wrapf(err, "disabling addon %s", addon.AssetName) } } diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index d882492ddfc0..6728192edbdf 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -377,3 +377,14 @@ func addMinikubeDirToAssets(basedir, vmpath string, assets *[]CopyableFile) erro } return nil } + +// GenerateTemplateData generates template data for template assets +func GenerateTemplateData(cfg config.KubernetesConfig) interface{} { + opts := struct { + ImageRepository string + }{ + ImageRepository: cfg.ImageRepository, + } + + return opts +} diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index e4ec43e60549..b5b95415ba45 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -432,7 +432,7 @@ func (k *Bootstrapper) UpdateCluster(cfg config.KubernetesConfig) error { if err != nil { return errors.Wrap(err, "runtime") } - kubeadmCfg, opts, err := generateConfig(cfg, r) + kubeadmCfg, err := generateConfig(cfg, r) if err != nil { return errors.Wrap(err, "generating kubeadm cfg") } @@ -480,7 +480,7 @@ func (k *Bootstrapper) UpdateCluster(cfg config.KubernetesConfig) error { return errors.Wrap(err, "downloading binaries") } - if err := addAddons(&files, opts); err != nil { + if err := addAddons(&files, assets.GenerateTemplateData(cfg)); err != nil { return errors.Wrap(err, "adding addons") } @@ -501,22 +501,22 @@ sudo systemctl start kubelet return nil } -func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, interface{}, error) { +func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, error) { version, err := ParseKubernetesVersion(k8s.KubernetesVersion) if err != nil { - return "", nil, errors.Wrap(err, "parsing kubernetes version") + return "", errors.Wrap(err, "parsing kubernetes version") } // parses a map of the feature gates for kubeadm and component kubeadmFeatureArgs, componentFeatureArgs, err := ParseFeatureArgs(k8s.FeatureGates) if err != nil { - return "", nil, errors.Wrap(err, "parses feature gate config for kubeadm and component") + return "", errors.Wrap(err, "parses feature gate config for kubeadm and component") } // generates a map of component to extra args for apiserver, controller-manager, and scheduler extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version, componentFeatureArgs) if err != nil { - return "", nil, errors.Wrap(err, "generating extra component config for kubeadm") + return "", errors.Wrap(err, "generating extra component config for kubeadm") } // In case of no port assigned, use util.APIServerPort @@ -571,10 +571,10 @@ func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, in configTmpl = configTmplV1Beta1 } if err := configTmpl.Execute(&b, opts); err != nil { - return "", nil, err + return "", err } - return b.String(), opts, nil + return b.String(), nil } func maybeDownloadAndCache(binary, version string) (string, error) { diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go index 35cbb49bf681..6e777e3c7a92 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go @@ -118,11 +118,10 @@ ExecStart=/usr/bin/kubelet --allow-privileged=true --authorization-mode=Webhook func TestGenerateConfig(t *testing.T) { tests := []struct { - description string - cfg config.KubernetesConfig - expectedCfg string - expectedOpts interface{} - shouldErr bool + description string + cfg config.KubernetesConfig + expectedCfg string + shouldErr bool }{ { description: "no extra args", @@ -148,37 +147,6 @@ nodeName: minikube apiServerExtraArgs: admission-control: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" `, - expectedOpts: struct { - CertDir string - ServiceCIDR string - AdvertiseAddress string - APIServerPort int - KubernetesVersion string - EtcdDataDir string - NodeName string - CRISocket string - ImageRepository string - ExtraArgs []ComponentExtraArgs - FeatureArgs map[string]bool - NoTaintMaster bool - }{ - CertDir: util.DefaultCertPath, - ServiceCIDR: util.DefaultServiceCIDR, - AdvertiseAddress: "192.168.1.100", - APIServerPort: 8443, - KubernetesVersion: "v1.10.0", - EtcdDataDir: "/data/minikube", - NodeName: "minikube", - ExtraArgs: []ComponentExtraArgs{ - { - Component: "apiServer", - Options: map[string]string{ - "admission-control": "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", - }, - }, - }, - FeatureArgs: map[string]bool{}, - NoTaintMaster: true}, }, { description: "extra args all components", @@ -471,38 +439,6 @@ imageRepository: docker-proxy-image.io/google_containers apiServerExtraArgs: admission-control: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" `, - expectedOpts: struct { - CertDir string - ServiceCIDR string - AdvertiseAddress string - APIServerPort int - KubernetesVersion string - EtcdDataDir string - NodeName string - CRISocket string - ImageRepository string - ExtraArgs []ComponentExtraArgs - FeatureArgs map[string]bool - NoTaintMaster bool - }{ - CertDir: util.DefaultCertPath, - ServiceCIDR: util.DefaultServiceCIDR, - AdvertiseAddress: "192.168.1.100", - APIServerPort: 8443, - KubernetesVersion: "v1.10.0", - EtcdDataDir: "/data/minikube", - NodeName: "minikube", - ExtraArgs: []ComponentExtraArgs{ - { - Component: "apiServer", - Options: map[string]string{ - "admission-control": "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", - }, - }, - }, - ImageRepository: "docker-proxy-image.io/google_containers", - FeatureArgs: map[string]bool{}, - NoTaintMaster: true}, }, } @@ -513,7 +449,7 @@ apiServerExtraArgs: } t.Run(test.description, func(t *testing.T) { - gotCfg, gotOpts, err := generateConfig(test.cfg, runtime) + gotCfg, err := generateConfig(test.cfg, runtime) if err != nil && !test.shouldErr { t.Errorf("got unexpected error generating config: %v", err) return @@ -529,11 +465,6 @@ apiServerExtraArgs: if diff := cmp.Diff(gotSplit, wantSplit); diff != "" { t.Errorf("unexpected diff: (-want +got)\n%s\ngot: %s\n", diff, gotCfg) } - if test.expectedOpts != nil { - if diff := cmp.Diff(test.expectedOpts, gotOpts); diff != "" { - t.Errorf("opts differ: (-want +got)\n%s", diff) - } - } }) } } diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index a31f657f3e73..3972ea318ebf 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -202,7 +202,7 @@ const ( // DefaultMsize is the default number of bytes to use for 9p packet payload DefaultMsize = 262144 // DefaultMountVersion is the default 9p version to use for mount - DefaultMountVersion = "9p2000.L" + DefaultMountVersion = "9p2000.L" ) // GetKubernetesReleaseURL gets the location of a kubernetes client diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 83cbc67f8b8d..6b2ec9b437ec 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -191,6 +191,7 @@ func (m *MinikubeRunner) RunDaemon2(command string) (*exec.Cmd, *bufio.Reader, * } return cmd, bufio.NewReader(stdoutPipe), bufio.NewReader(stderrPipe) } + // SSH returns the output of running a command using SSH func (m *MinikubeRunner) SSH(command string) (string, error) { path, _ := filepath.Abs(m.BinaryPath)