diff --git a/Makefile b/Makefile index 9069ef5c7317..7ed527f3e6dc 100755 --- a/Makefile +++ b/Makefile @@ -473,7 +473,7 @@ $(ISO_BUILD_IMAGE): deploy/iso/minikube-iso/Dockerfile @echo "$(@) successfully built" out/storage-provisioner: - GOOS=linux go build -o $@ -ldflags=$(PROVISIONER_LDFLAGS) cmd/storage-provisioner/main.go + CGO_ENABLED=0 GOOS=linux go build -o $@ -ldflags=$(PROVISIONER_LDFLAGS) cmd/storage-provisioner/main.go .PHONY: storage-provisioner-image storage-provisioner-image: out/storage-provisioner ## Build storage-provisioner docker image @@ -483,6 +483,13 @@ else docker build -t $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG) -f deploy/storage-provisioner/Dockerfile-$(GOARCH) . endif +.PHONY: kic-base-image +kic-base-image: ## builds the base image used for kic. + docker rmi -f $(REGISTRY)/kicbase:v0.0.1-snapshot || true + docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.1-snapshot --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) . + + + .PHONY: push-storage-provisioner-image push-storage-provisioner-image: storage-provisioner-image ## Push storage-provisioner docker image using gcloud ifeq ($(GOARCH),amd64) @@ -591,6 +598,7 @@ out/mkcmp: out/performance-monitor: GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/monitor/monitor.go + .PHONY: help help: @printf "\033[1mAvailable targets for minikube ${VERSION}\033[21m\n" diff --git a/README.md b/README.md index e59cce2657a8..75423b747468 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ minikube runs the latest stable release of Kubernetes, with support for standard * Multi-cluster - using `minikube start -p ` * NodePorts - using `minikube service` * [Persistent Volumes](https://minikube.sigs.k8s.io/docs/reference/persistent_volumes/) -* Ingress +* [Ingress](https://kubernetes.io/docs/tasks/access-application-cluster/ingress-minikube/) * [Dashboard](https://minikube.sigs.k8s.io/docs/tasks/dashboard/) - `minikube dashboard` * [Container runtimes](https://minikube.sigs.k8s.io/docs/reference/runtimes/) - `start --container-runtime` * [Configure apiserver and kubelet options](https://minikube.sigs.k8s.io/docs/reference/configuration/kubernetes/) via command-line flags diff --git a/cmd/minikube/cmd/config/addons_list.go b/cmd/minikube/cmd/config/addons_list.go index 08492a367836..18a5ff8016a1 100644 --- a/cmd/minikube/cmd/config/addons_list.go +++ b/cmd/minikube/cmd/config/addons_list.go @@ -22,17 +22,17 @@ import ( "os" "sort" "strings" - "text/template" + "github.com/golang/glog" + "github.com/olekukonko/tablewriter" "github.com/spf13/cobra" + "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/out" ) -const defaultAddonListFormat = "- {{.AddonName}}: {{.AddonStatus}}\n" - -var addonListFormat string var addonListOutput string // AddonListTemplate represents the addon list template @@ -50,10 +50,6 @@ var addonsListCmd = &cobra.Command{ exit.UsageT("usage: minikube addons list") } - if addonListOutput != "list" && addonListFormat != defaultAddonListFormat { - exit.UsageT("Cannot use both --output and --format options") - } - switch strings.ToLower(addonListOutput) { case "list": printAddonsList() @@ -66,14 +62,6 @@ var addonsListCmd = &cobra.Command{ } func init() { - addonsListCmd.Flags().StringVarP( - &addonListFormat, - "format", - "f", - defaultAddonListFormat, - `Go template format string for the addon list output. The format for Go templates can be found here: https://golang.org/pkg/text/template/ -For the list of accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd/config#AddonListTemplate`) - addonsListCmd.Flags().StringVarP( &addonListOutput, "output", @@ -84,6 +72,13 @@ For the list of accessible variables for the template, see the struct values her AddonsCmd.AddCommand(addonsListCmd) } +var iconFromStatus = func(addonStatus bool) string { + if addonStatus { + return "✅" + } + return " " // because emoji indentation is different +} + var stringFromStatus = func(addonStatus bool) string { if addonStatus { return "enabled" @@ -97,6 +92,13 @@ var printAddonsList = func() { addonNames = append(addonNames, addonName) } sort.Strings(addonNames) + var tData [][]string + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader([]string{"Addon Name", "Profile", "Status"}) + table.SetAutoFormatHeaders(true) + table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true}) + table.SetCenterSeparator("|") + pName := viper.GetString(config.MachineProfile) for _, addonName := range addonNames { addonBundle := assets.Addons[addonName] @@ -104,20 +106,25 @@ var printAddonsList = func() { if err != nil { exit.WithError("Error getting addons status", err) } - tmpl, err := template.New("list").Parse(addonListFormat) - if err != nil { - exit.WithError("Error creating list template", err) - } - listTmplt := AddonListTemplate{addonName, stringFromStatus(addonStatus)} - err = tmpl.Execute(os.Stdout, listTmplt) - if err != nil { - exit.WithError("Error executing list template", err) - } + tData = append(tData, []string{addonName, pName, fmt.Sprintf("%s %s", stringFromStatus(addonStatus), iconFromStatus(addonStatus))}) } + + table.AppendBulk(tData) + table.Render() + + v, _, err := config.ListProfiles() + if err != nil { + glog.Infof("error getting list of porfiles: %v", err) + } + if len(v) > 1 { + out.T(out.Tip, "To see addons list for other profiles use: `minikube addons -p name list`") + } + } var printAddonsJSON = func() { addonNames := make([]string, 0, len(assets.Addons)) + pName := viper.GetString(config.MachineProfile) for addonName := range assets.Addons { addonNames = append(addonNames, addonName) } @@ -134,7 +141,8 @@ var printAddonsJSON = func() { } addonsMap[addonName] = map[string]interface{}{ - "Status": stringFromStatus(addonStatus), + "Status": stringFromStatus(addonStatus), + "Profile": pName, } } jsonString, _ := json.Marshal(addonsMap) diff --git a/cmd/minikube/cmd/config/config.go b/cmd/minikube/cmd/config/config.go index 34579142a915..7bf19d118d96 100644 --- a/cmd/minikube/cmd/config/config.go +++ b/cmd/minikube/cmd/config/config.go @@ -142,123 +142,8 @@ var settings = []Setting{ set: SetBool, }, { - name: "dashboard", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "istio-provisioner", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "istio", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "addon-manager", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "default-storageclass", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableStorageClasses}, - }, - { - name: "efk", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "ingress", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "insecure-registry", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "registry", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "registry-creds", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "freshpod", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "storage-provisioner", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "storage-provisioner-gluster", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableStorageClasses}, - }, - { - name: "metrics-server", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "nvidia-driver-installer", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "nvidia-gpu-device-plugin", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "logviewer", - set: SetBool, - validations: []setFn{IsValidAddon}, - }, - { - name: "gvisor", - set: SetBool, - validations: []setFn{IsValidAddon, IsContainerdRuntime}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "helm-tiller", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, - }, - { - name: "ingress-dns", - set: SetBool, - validations: []setFn{IsValidAddon}, - callbacks: []setFn{EnableOrDisableAddon}, + name: "insecure-registry", + set: SetString, }, { name: "hyperv-virtual-switch", diff --git a/cmd/minikube/cmd/config/disable.go b/cmd/minikube/cmd/config/disable.go index 6901f43bd416..c6c538f4b101 100644 --- a/cmd/minikube/cmd/config/disable.go +++ b/cmd/minikube/cmd/config/disable.go @@ -18,6 +18,9 @@ package config import ( "github.com/spf13/cobra" + "github.com/spf13/viper" + "k8s.io/minikube/pkg/addons" + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/out" ) @@ -32,7 +35,7 @@ var addonsDisableCmd = &cobra.Command{ } addon := args[0] - err := Set(addon, "false") + err := addons.Set(addon, "false", viper.GetString(config.MachineProfile)) if err != nil { exit.WithError("disable failed", err) } diff --git a/cmd/minikube/cmd/config/disable_test.go b/cmd/minikube/cmd/config/disable_test.go deleted file mode 100644 index dbde3850c751..000000000000 --- a/cmd/minikube/cmd/config/disable_test.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "testing" - - "gotest.tools/assert" - pkgConfig "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/localpath" -) - -func TestDisableUnknownAddon(t *testing.T) { - if err := Set("InvalidAddon", "false"); err == nil { - t.Fatalf("Disable did not return error for unknown addon") - } -} - -func TestDisableAddon(t *testing.T) { - if err := Set("dashboard", "false"); err != nil { - t.Fatalf("Disable returned unexpected error: " + err.Error()) - } - config, _ := pkgConfig.ReadConfig(localpath.ConfigFile) - assert.Equal(t, config["dashboard"], false) -} diff --git a/cmd/minikube/cmd/config/enable.go b/cmd/minikube/cmd/config/enable.go index 20b018b135e2..a920b9be0d77 100644 --- a/cmd/minikube/cmd/config/enable.go +++ b/cmd/minikube/cmd/config/enable.go @@ -18,6 +18,9 @@ package config import ( "github.com/spf13/cobra" + "github.com/spf13/viper" + "k8s.io/minikube/pkg/addons" + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/out" ) @@ -32,7 +35,7 @@ var addonsEnableCmd = &cobra.Command{ } addon := args[0] - err := Set(addon, "true") + err := addons.Set(addon, "true", viper.GetString(config.MachineProfile)) if err != nil { exit.WithError("enable failed", err) } diff --git a/cmd/minikube/cmd/config/get_test.go b/cmd/minikube/cmd/config/get_test.go new file mode 100644 index 000000000000..9f14bbf7d3b7 --- /dev/null +++ b/cmd/minikube/cmd/config/get_test.go @@ -0,0 +1,41 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import "testing" + +func TestGetNotFound(t *testing.T) { + _, err := Get("nonexistent") + if err == nil { + t.Fatalf("Get did not return error for unknown property") + } +} + +func TestGetOK(t *testing.T) { + name := "vm-driver" + err := Set(name, "virtualbox") + if err != nil { + t.Fatalf("Set returned error for property %s", name) + } + val, err := Get(name) + if err != nil { + t.Fatalf("Get returned error for property %s", name) + } + if val != "virtualbox" { + t.Fatalf("Get returned %s, expected virtualbox", val) + } +} diff --git a/cmd/minikube/cmd/config/profile_list.go b/cmd/minikube/cmd/config/profile_list.go index 02f8b9ba1d98..04207ddca173 100644 --- a/cmd/minikube/cmd/config/profile_list.go +++ b/cmd/minikube/cmd/config/profile_list.go @@ -59,7 +59,6 @@ var profileListCmd = &cobra.Command{ var printProfilesTable = func() { var validData [][]string - table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Profile", "VM Driver", "NodeIP", "Node Port", "Kubernetes Version", "Status"}) table.SetAutoFormatHeaders(false) diff --git a/cmd/minikube/cmd/config/set_test.go b/cmd/minikube/cmd/config/set_test.go index e027f7aebbd9..9a6adde38412 100644 --- a/cmd/minikube/cmd/config/set_test.go +++ b/cmd/minikube/cmd/config/set_test.go @@ -24,3 +24,24 @@ func TestNotFound(t *testing.T) { t.Fatalf("Set did not return error for unknown property") } } + +func TestSetNotAllowed(t *testing.T) { + err := Set("vm-driver", "123456") + if err == nil || err.Error() != "[driver \"123456\" is not supported]" { + t.Fatalf("Set did not return error for unallowed value") + } +} + +func TestSetOK(t *testing.T) { + err := Set("vm-driver", "virtualbox") + if err != nil { + t.Fatalf("Set returned error for valid property value") + } + val, err := Get("vm-driver") + if err != nil { + t.Fatalf("Get returned error for valid property") + } + if val != "virtualbox" { + t.Fatalf("Get returned %s, expected \"virtualbox\"", val) + } +} diff --git a/cmd/minikube/cmd/config/util.go b/cmd/minikube/cmd/config/util.go index dc8d9e88bbf1..0546590d5d94 100644 --- a/cmd/minikube/cmd/config/util.go +++ b/cmd/minikube/cmd/config/util.go @@ -18,26 +18,13 @@ package config import ( "fmt" - "os" "strconv" "strings" - "github.com/pkg/errors" - "github.com/spf13/viper" - "k8s.io/minikube/pkg/minikube/assets" - "k8s.io/minikube/pkg/minikube/cluster" - "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/exit" - "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" - "k8s.io/minikube/pkg/minikube/storageclass" - pkgutil "k8s.io/minikube/pkg/util" ) -// defaultStorageClassProvisioner is the name of the default storage class provisioner -const defaultStorageClassProvisioner = "standard" - // Runs all the validation or callback functions and collects errors func run(name string, value string, fns []setFn) error { var errors []error @@ -107,155 +94,6 @@ func SetBool(m config.MinikubeConfig, name string, val string) error { return nil } -// EnableOrDisableAddon updates addon status executing any commands necessary -func EnableOrDisableAddon(name string, val string) error { - enable, err := strconv.ParseBool(val) - if err != nil { - return errors.Wrapf(err, "parsing bool: %s", name) - } - addon := assets.Addons[name] - - // check addon status before enabling/disabling it - alreadySet, err := isAddonAlreadySet(addon, enable) - if err != nil { - out.ErrT(out.Conflict, "{{.error}}", out.V{"error": err}) - return err - } - //if addon is already enabled or disabled, do nothing - if alreadySet { - return nil - } - - if name == "istio" && enable { - minMem := 8192 - minCpus := 4 - memorySizeMB := pkgutil.CalculateSizeInMB(viper.GetString("memory")) - cpuCount := viper.GetInt("cpus") - if memorySizeMB < minMem || cpuCount < minCpus { - out.WarningT("Enable istio needs {{.minMem}} MB of memory and {{.minCpus}} CPUs.", out.V{"minMem": minMem, "minCpus": minCpus}) - } - } - - // TODO(r2d4): config package should not reference API, pull this out - api, err := machine.NewAPIClient() - if err != nil { - return errors.Wrap(err, "machine client") - } - defer api.Close() - - //if minikube is not running, we return and simply update the value in the addon - //config and rewrite the file - if !cluster.IsMinikubeRunning(api) { - return nil - } - - cfg, err := config.Load(viper.GetString(config.MachineProfile)) - if err != nil && !os.IsNotExist(err) { - exit.WithCodeT(exit.Data, "Unable to load config: {{.error}}", out.V{"error": err}) - } - - host, err := cluster.CheckIfHostExistsAndLoad(api, cfg.Name) - if err != nil { - return errors.Wrap(err, "getting host") - } - - cmd, err := machine.CommandRunner(host) - if err != nil { - return errors.Wrap(err, "command runner") - } - - data := assets.GenerateTemplateData(cfg.KubernetesConfig) - return enableOrDisableAddonInternal(addon, cmd, data, enable) -} - -func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) { - addonStatus, err := addon.IsEnabled() - - if err != nil { - return false, errors.Wrap(err, "get the addon status") - } - - if addonStatus && enable { - return true, nil - } else if !addonStatus && !enable { - return true, nil - } - - return false, nil -} - -func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool) error { - var err error - - if enable { - for _, addon := range addon.Assets { - var addonFile assets.CopyableFile - if addon.IsTemplate() { - addonFile, err = addon.Evaluate(data) - if err != nil { - return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName()) - } - - } else { - addonFile = addon - } - if err := cmd.Copy(addonFile); err != nil { - return errors.Wrapf(err, "enabling addon %s", addon.AssetName) - } - } - } else { - for _, addon := range addon.Assets { - var addonFile assets.CopyableFile - if addon.IsTemplate() { - addonFile, err = addon.Evaluate(data) - if err != nil { - return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName()) - } - - } else { - addonFile = addon - } - if err := cmd.Remove(addonFile); err != nil { - return errors.Wrapf(err, "disabling addon %s", addon.AssetName) - } - } - } - return nil -} - -// EnableOrDisableStorageClasses enables or disables storage classes -func EnableOrDisableStorageClasses(name, val string) error { - enable, err := strconv.ParseBool(val) - if err != nil { - return errors.Wrap(err, "Error parsing boolean") - } - - class := defaultStorageClassProvisioner - if name == "storage-provisioner-gluster" { - class = "glusterfile" - } - storagev1, err := storageclass.GetStoragev1() - if err != nil { - return errors.Wrapf(err, "Error getting storagev1 interface %v ", err) - } - - if enable { - // Only StorageClass for 'name' should be marked as default - err = storageclass.SetDefaultStorageClass(storagev1, class) - if err != nil { - return errors.Wrapf(err, "Error making %s the default storage class", class) - } - } else { - // Unset the StorageClass as default - err := storageclass.DisableDefaultStorageClass(storagev1, class) - if err != nil { - return errors.Wrapf(err, "Error disabling %s as the default storage class", class) - } - } - - return EnableOrDisableAddon(name, val) -} - // ErrValidateProfile Error to validate profile type ErrValidateProfile struct { Name string diff --git a/cmd/minikube/cmd/config/util_test.go b/cmd/minikube/cmd/config/util_test.go index f1627f19fa82..1310b95cbf7d 100644 --- a/cmd/minikube/cmd/config/util_test.go +++ b/cmd/minikube/cmd/config/util_test.go @@ -20,7 +20,6 @@ import ( "fmt" "testing" - "k8s.io/minikube/pkg/minikube/assets" pkgConfig "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/driver" ) @@ -83,37 +82,6 @@ func TestSetBool(t *testing.T) { } } -func TestIsAddonAlreadySet(t *testing.T) { - testCases := []struct { - addonName string - }{ - { - addonName: "ingress", - }, - - { - addonName: "registry", - }, - } - - for _, test := range testCases { - addon := assets.Addons[test.addonName] - addonStatus, _ := addon.IsEnabled() - - alreadySet, err := isAddonAlreadySet(addon, addonStatus) - if !alreadySet { - if addonStatus { - t.Errorf("Did not get expected status, \n\n expected %+v already enabled", test.addonName) - } else { - t.Errorf("Did not get expected status, \n\n expected %+v already disabled", test.addonName) - } - } - if err != nil { - t.Errorf("Got unexpected error: %+v", err) - } - } -} - func TestValidateProfile(t *testing.T) { testCases := []struct { profileName string diff --git a/cmd/minikube/cmd/config/validations.go b/cmd/minikube/cmd/config/validations.go index 4220afb1235e..60ec6a53ab96 100644 --- a/cmd/minikube/cmd/config/validations.go +++ b/cmd/minikube/cmd/config/validations.go @@ -25,25 +25,11 @@ import ( "strings" units "github.com/docker/go-units" - "github.com/pkg/errors" - "github.com/spf13/viper" - "k8s.io/minikube/pkg/minikube/assets" - "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/out" ) -// containerdOnlyMsg is the message shown when a containerd-only addon is enabled -const containerdOnlyAddonMsg = ` -This addon can only be enabled with the containerd runtime backend. To enable this backend, please first stop minikube with: - -minikube stop - -and then start minikube again with the following flags: - -minikube start --container-runtime=containerd --docker-opt containerd=/var/run/containerd/containerd.sock` - // IsValidDriver checks if a driver is supported func IsValidDriver(string, name string) error { if driver.Supported(name) { @@ -140,14 +126,6 @@ func IsValidPath(name string, path string) error { return nil } -// IsValidAddon checks if a string is a valid addon -func IsValidAddon(name string, val string) error { - if _, ok := assets.Addons[name]; ok { - return nil - } - return errors.Errorf("Cannot enable/disable invalid addon %s", name) -} - // IsValidRuntime checks if a string is a valid runtime func IsValidRuntime(name string, runtime string) error { _, err := cruntime.New(cruntime.Config{Type: runtime}) @@ -156,20 +134,3 @@ func IsValidRuntime(name string, runtime string) error { } return nil } - -// IsContainerdRuntime is a validator which returns an error if the current runtime is not containerd -func IsContainerdRuntime(_, _ string) error { - config, err := config.Load(viper.GetString(config.MachineProfile)) - if err != nil { - return fmt.Errorf("config.Load: %v", err) - } - r, err := cruntime.New(cruntime.Config{Type: config.KubernetesConfig.ContainerRuntime}) - if err != nil { - return err - } - _, ok := r.(*cruntime.Containerd) - if !ok { - return fmt.Errorf(containerdOnlyAddonMsg) - } - return nil -} diff --git a/cmd/minikube/cmd/dashboard.go b/cmd/minikube/cmd/dashboard.go index f1eaefda34d1..bdcc2042090c 100644 --- a/cmd/minikube/cmd/dashboard.go +++ b/cmd/minikube/cmd/dashboard.go @@ -33,7 +33,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/viper" - configcmd "k8s.io/minikube/cmd/minikube/cmd/config" + pkgaddons "k8s.io/minikube/pkg/addons" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/cluster" "k8s.io/minikube/pkg/minikube/config" @@ -106,7 +106,7 @@ var dashboardCmd = &cobra.Command{ // Send status messages to stderr for folks re-using this output. out.ErrT(out.Enabling, "Enabling dashboard ...") // Enable the dashboard add-on - err = configcmd.Set("dashboard", "true") + err = pkgaddons.Set("dashboard", "true", viper.GetString(config.MachineProfile)) if err != nil { exit.WithError("Unable to enable dashboard", err) } diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index a3ff0039851b..55325fee8055 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -46,6 +46,7 @@ import ( "github.com/spf13/viper" "golang.org/x/sync/errgroup" cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" + pkgaddons "k8s.io/minikube/pkg/addons" "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" "k8s.io/minikube/pkg/minikube/bootstrapper/images" @@ -115,6 +116,7 @@ const ( hostDNSResolver = "host-dns-resolver" waitUntilHealthy = "wait" force = "force" + dryRun = "dry-run" interactive = "interactive" waitTimeout = "wait-timeout" nativeSSH = "native-ssh" @@ -157,6 +159,7 @@ func initMinikubeFlags() { startCmd.Flags().Bool(force, false, "Force minikube to perform possibly dangerous operations") startCmd.Flags().Bool(interactive, true, "Allow user prompts for more information") + startCmd.Flags().Bool(dryRun, false, "dry-run mode. Validates configuration, but does does not mutate system state") startCmd.Flags().Int(cpus, 2, "Number of CPUs allocated to the minikube VM.") startCmd.Flags().String(memory, defaultMemorySize, "Amount of RAM allocated to the minikube VM (format: [], where unit = b, k, m or g).") @@ -231,7 +234,7 @@ func initNetworkingFlags() { startCmd.Flags().StringSliceVar(®istryMirror, "registry-mirror", nil, "Registry mirrors to pass to the Docker daemon") startCmd.Flags().String(imageRepository, "", "Alternative image repository to pull docker images from. This can be used when you have limited access to gcr.io. Set it to \"auto\" to let minikube decide one for you. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers") startCmd.Flags().String(imageMirrorCountry, "", "Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.") - startCmd.Flags().String(serviceCIDR, pkgutil.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.") + startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.") startCmd.Flags().StringArrayVar(&dockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)") startCmd.Flags().StringArrayVar(&dockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)") } @@ -267,7 +270,11 @@ func platform() string { // This environment is exotic, let's output a bit more. if vrole == "guest" || runtime.GOARCH != "amd64" { - s.WriteString(fmt.Sprintf(" (%s/%s)", vsys, runtime.GOARCH)) + if vsys != "" { + s.WriteString(fmt.Sprintf(" (%s/%s)", vsys, runtime.GOARCH)) + } else { + s.WriteString(fmt.Sprintf(" (%s)", runtime.GOARCH)) + } } return s.String() } @@ -305,7 +312,9 @@ func runStart(cmd *cobra.Command, args []string) { validateUser(driverName) // Download & update the driver, even in --download-only mode - updateDriver(driverName) + if !viper.GetBool(dryRun) { + updateDriver(driverName) + } k8sVersion, isUpgrade := getKubernetesVersion(existing) config, err := generateCfgFromFlags(cmd, k8sVersion, driverName) @@ -313,12 +322,14 @@ func runStart(cmd *cobra.Command, args []string) { exit.WithError("Failed to generate config", err) } - if !driver.BareMetal(driverName) && !driver.IsKIC(driverName) { - if err := cluster.CacheISO(config); err != nil { - exit.WithError("Failed to cache ISO", err) - } + // This is about as far as we can go without overwriting config files + if viper.GetBool(dryRun) { + out.T(out.DryRun, `dry-run validation complete!`) + return } + cacheISO(&config, driverName) + if viper.GetBool(nativeSSH) { ssh.SetDefaultClient(ssh.Native) } else { @@ -389,9 +400,17 @@ func updateDriver(driverName string) { } } +func cacheISO(config *cfg.MachineConfig, driverName string) { + if !driver.BareMetal(driverName) && !driver.IsKIC(driverName) { + if err := cluster.CacheISO(*config); err != nil { + exit.WithError("Failed to cache ISO", err) + } + } +} + func enableAddons() { for _, a := range addonList { - err := cmdcfg.Set(a, "true") + err := pkgaddons.Set(a, "true", viper.GetString(config.MachineProfile)) if err != nil { exit.WithError("addon enable failed", err) } @@ -425,16 +444,22 @@ func displayEnviron(env []string) { } func setupKubeconfig(h *host.Host, c *cfg.MachineConfig, clusterName string) (*kubeconfig.Settings, error) { - addr, err := h.Driver.GetURL() - if err != nil { - exit.WithError("Failed to get driver URL", err) + addr := "" + var err error + if driver.IsKIC(h.DriverName) { + addr = fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(c.KubernetesConfig.NodePort))) + } else { + addr, err = h.Driver.GetURL() + if err != nil { + exit.WithError("Failed to get driver URL", err) + } + addr = strings.Replace(addr, "tcp://", "https://", -1) + addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1) } - addr = strings.Replace(addr, "tcp://", "https://", -1) - addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1) + if c.KubernetesConfig.APIServerName != constants.APIServerName { addr = strings.Replace(addr, c.KubernetesConfig.NodeIP, c.KubernetesConfig.APIServerName, -1) } - kcs := &kubeconfig.Settings{ ClusterName: clusterName, ClusterServerAddress: addr, @@ -554,6 +579,8 @@ func showKubectlInfo(kcs *kubeconfig.Settings, k8sVersion string, machineName st func selectDriver(existing *cfg.MachineConfig) string { name := viper.GetString("vm-driver") glog.Infof("selectDriver: flag=%q, old=%v", name, existing) + + driver.SetLibvirtURI(viper.GetString(kvmQemuURI)) options := driver.Choices() pick, alts := driver.Choose(name, options) @@ -704,7 +731,7 @@ func minikubeCmd() string { return "minikube" } -// validerUser validates minikube is run by the recommended user (privileged or regular) +// validateUser validates minikube is run by the recommended user (privileged or regular) func validateUser(drvName string) { u, err := user.Current() if err != nil { @@ -738,23 +765,50 @@ func validateUser(drvName string) { } } -// validateFlags validates the supplied flags against known bad combinations -func validateFlags(cmd *cobra.Command, drvName string) { +// validateDiskSize validates the disk size matches the minimum recommended +func validateDiskSize() { diskSizeMB := pkgutil.CalculateSizeInMB(viper.GetString(humanReadableDiskSize)) if diskSizeMB < pkgutil.CalculateSizeInMB(minimumDiskSize) && !viper.GetBool(force) { exit.WithCodeT(exit.Config, "Requested disk size {{.requested_size}} is less than minimum of {{.minimum_size}}", out.V{"requested_size": diskSizeMB, "minimum_size": pkgutil.CalculateSizeInMB(minimumDiskSize)}) } +} +// validateMemorySize validates the memory size matches the minimum recommended +func validateMemorySize() { memorySizeMB := pkgutil.CalculateSizeInMB(viper.GetString(memory)) if memorySizeMB < pkgutil.CalculateSizeInMB(minimumMemorySize) && !viper.GetBool(force) { - exit.UsageT("Requested memory allocation {{.requested_size}} is less than the minimum allowed of {{.minimum_size}}", out.V{"requested_size": memorySizeMB, "minimum_size": pkgutil.CalculateSizeInMB(minimumMemorySize)}) + exit.WithCodeT(exit.Config, "Requested memory allocation {{.requested_size}} is less than the minimum allowed of {{.minimum_size}}", out.V{"requested_size": memorySizeMB, "minimum_size": pkgutil.CalculateSizeInMB(minimumMemorySize)}) } if memorySizeMB < pkgutil.CalculateSizeInMB(defaultMemorySize) && !viper.GetBool(force) { out.T(out.Notice, "Requested memory allocation ({{.memory}}MB) is less than the default memory allocation of {{.default_memorysize}}MB. Beware that minikube might not work correctly or crash unexpectedly.", out.V{"memory": memorySizeMB, "default_memorysize": pkgutil.CalculateSizeInMB(defaultMemorySize)}) } +} +// validateCPUCount validates the cpu count matches the minimum recommended +func validateCPUCount(local bool) { var cpuCount int + if local { + // Uses the gopsutil cpu package to count the number of physical cpu cores + ci, err := cpu.Counts(false) + if err != nil { + glog.Warningf("Unable to get CPU info: %v", err) + } else { + cpuCount = ci + } + } else { + cpuCount = viper.GetInt(cpus) + } + if cpuCount < minimumCPUS && !viper.GetBool(force) { + exit.UsageT("Requested cpu count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}", out.V{"requested_cpus": cpuCount, "minimum_cpus": minimumCPUS}) + } +} + +// validateFlags validates the supplied flags against known bad combinations +func validateFlags(cmd *cobra.Command, drvName string) { + validateDiskSize() + validateMemorySize() + if driver.BareMetal(drvName) { if viper.GetString(cfg.MachineProfile) != constants.DefaultMachineName { exit.WithCodeT(exit.Config, "The 'none' driver does not support multiple profiles: https://minikube.sigs.k8s.io/docs/reference/drivers/none/") @@ -771,21 +825,10 @@ func validateFlags(cmd *cobra.Command, drvName string) { if runtime != "docker" { out.WarningT("Using the '{{.runtime}}' runtime with the 'none' driver is an untested configuration!", out.V{"runtime": runtime}) } - - // Uses the gopsutil cpu package to count the number of physical cpu cores - ci, err := cpu.Counts(false) - if err != nil { - glog.Warningf("Unable to get CPU info: %v", err) - } else { - cpuCount = ci - } - } else { - cpuCount = viper.GetInt(cpus) - } - if cpuCount < minimumCPUS && !viper.GetBool(force) { - exit.UsageT("Requested cpu count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}", out.V{"requested_cpus": cpuCount, "minimum_cpus": minimumCPUS}) } + validateCPUCount(driver.BareMetal(drvName)) + // check that kubeadm extra args contain only whitelisted parameters for param := range extraOptions.AsMap().Get(bsutil.Kubeadm) { if !cfg.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmCmdParam], param) && @@ -959,29 +1002,35 @@ func setDockerProxy() { } // autoSetDriverOptions sets the options needed for specific vm-driver automatically. -func autoSetDriverOptions(cmd *cobra.Command, drvName string) error { +func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) { + err = nil hints := driver.FlagDefaults(drvName) - if !cmd.Flags().Changed("extra-config") && hints.ExtraOptions != "" { - return extraOptions.Set(hints.ExtraOptions) + if !cmd.Flags().Changed("extra-config") && len(hints.ExtraOptions) > 0 { + for _, eo := range hints.ExtraOptions { + glog.Infof("auto setting extra-config to %q.", eo) + err = extraOptions.Set(eo) + if err != nil { + err = errors.Wrapf(err, "setting extra option %s", eo) + } + } } if !cmd.Flags().Changed(cacheImages) { viper.Set(cacheImages, hints.CacheImages) } - // currently only used for kic if !cmd.Flags().Changed(containerRuntime) && hints.ContainerRuntime != "" { viper.Set(containerRuntime, hints.ContainerRuntime) - glog.Infof("auto set container runtime to %s for kic driver.", hints.ContainerRuntime) - + glog.Infof("auto set %s to %q.", containerRuntime, hints.ContainerRuntime) } - if !cmd.Flags().Changed("bootstrapper") && hints.Bootstrapper != "" { + + if !cmd.Flags().Changed(cmdcfg.Bootstrapper) && hints.Bootstrapper != "" { viper.Set(cmdcfg.Bootstrapper, hints.Bootstrapper) - glog.Infof("auto set bootstrapper to %s for kic driver.", hints.Bootstrapper) + glog.Infof("auto set %s to %q.", cmdcfg.Bootstrapper, hints.Bootstrapper) } - return nil + return err } // prepareNone prepares the user and host for the joy of the "none" driver @@ -1094,7 +1143,7 @@ Suggested workarounds: func tryLookup(r command.Runner) { // DNS check - if rr, err := r.RunCmd(exec.Command("nslookup", "-querytype=ns", "kubernetes.io")); err != nil { + if rr, err := r.RunCmd(exec.Command("nslookup", "kubernetes.io")); err != nil { glog.Warningf("%s failed: %v", rr.Args, err) out.WarningT("VM may be unable to resolve external DNS records") } @@ -1126,9 +1175,10 @@ func getKubernetesVersion(old *cfg.MachineConfig) (string, bool) { isUpgrade := false if paramVersion == "" { // if the user did not specify any version then ... - if old != nil { // .. use the old version from config + if old != nil { // .. use the old version from config (if any) paramVersion = old.KubernetesConfig.KubernetesVersion - } else { // .. otherwise use the default version + } + if paramVersion == "" { // .. otherwise use the default version paramVersion = constants.DefaultKubernetesVersion } } diff --git a/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/sysctl.d/disable-ipv6.conf b/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/sysctl.d/disable-ipv6.conf new file mode 100644 index 000000000000..30b2d9b487d7 --- /dev/null +++ b/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/sysctl.d/disable-ipv6.conf @@ -0,0 +1,2 @@ +net.ipv6.conf.all.disable_ipv6=1 +net.ipv6.conf.default.disable_ipv6=1 diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash index 9657fcbb3d45..2d50a5c329cd 100644 --- a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash +++ b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash @@ -9,3 +9,4 @@ sha256 f7041a92e2d3a4c341be8df58f1076ba57ecb5daa02b6c65e652530c5f242739 v1.15.0. sha256 6218a99877da9b9895e0088944731f5384803c15628d4b3c6b40ba1ddd39e052 v1.15.1.tar.gz sha256 70d4c746fe207422c78420dc4239768f485eea639a38c993c02872ec6305dd1d v1.15.2.tar.gz sha256 05f9614c4d5970b4662499b84c270b0ab953596ee863dcd09c9dc7a2d2f09789 v1.16.0.tar.gz +sha256 57e1ee990ef2d5af8b32c33a21b4998682608e3556dcf1d3349666f55e7d95b9 v1.16.1.tar.gz diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk index 1447c603bc4a..2f7c82a233e1 100644 --- a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk +++ b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk @@ -4,8 +4,8 @@ # ################################################################################ -CRIO_BIN_VERSION = v1.16.0 -CRIO_BIN_COMMIT = fa99ff4ae2aa45115bf3c6bb33db07191db2518e +CRIO_BIN_VERSION = v1.16.1 +CRIO_BIN_COMMIT = bf8fcf34c942ba973a4f694094b46f914c779c0a CRIO_BIN_SITE = https://github.com/cri-o/cri-o/archive CRIO_BIN_SOURCE = $(CRIO_BIN_VERSION).tar.gz CRIO_BIN_DEPENDENCIES = host-go libgpgme diff --git a/deploy/iso/minikube-iso/package/podman/podman.hash b/deploy/iso/minikube-iso/package/podman/podman.hash index b796462fb8a2..5c24b789c85a 100644 --- a/deploy/iso/minikube-iso/package/podman/podman.hash +++ b/deploy/iso/minikube-iso/package/podman/podman.hash @@ -10,3 +10,4 @@ sha256 17fdf68e85106d0848e89825e191198a4079bb6d9ca6dd3e415e5c010192db9e v1.4.0.t sha256 45eb7bccd81a1431b0c7a0697829c0bcc397048595d143fd91179b31d22a3c63 v1.4.1.tar.gz sha256 2e027c1b935f3a03f27ef7f17823ccf334607a17d033d4ce53a90b98294e7f68 v1.4.4.tar.gz sha256 61b44b739c485125f179044f7aa7dc58c820f771bce4ce495fa555a38dc68b57 v1.6.3.tar.gz +sha256 6e59821320b435543bc7554e73faa66d5956e4ad3f7e7f4ea03bebd6726758e9 v1.6.4.tar.gz diff --git a/deploy/iso/minikube-iso/package/podman/podman.mk b/deploy/iso/minikube-iso/package/podman/podman.mk index b2baf62c633e..cffc991b0c12 100644 --- a/deploy/iso/minikube-iso/package/podman/podman.mk +++ b/deploy/iso/minikube-iso/package/podman/podman.mk @@ -1,5 +1,5 @@ -PODMAN_VERSION = v1.6.3 -PODMAN_COMMIT = 9d087f6a766259ba53b224944f1b7b778035c370 +PODMAN_VERSION = v1.6.4 +PODMAN_COMMIT = 5cc92849f7fc9dd734ca2fd8f3ae8830b9a7eb26 PODMAN_SITE = https://github.com/containers/libpod/archive PODMAN_SOURCE = $(PODMAN_VERSION).tar.gz PODMAN_LICENSE = Apache-2.0 @@ -24,7 +24,7 @@ endef define PODMAN_BUILD_CMDS mkdir -p $(@D)/bin - $(PODMAN_BIN_ENV) $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) GIT_COMMIT=$(PODMAN_COMMIT) PREFIX=/usr podman + $(PODMAN_BIN_ENV) CIRRUS_TAG=$(PODMAN_VERSION) $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) GIT_COMMIT=$(PODMAN_COMMIT) PREFIX=/usr podman endef define PODMAN_INSTALL_TARGET_CMDS diff --git a/deploy/iso/minikube-iso/package/runc-master/runc-master.mk b/deploy/iso/minikube-iso/package/runc-master/runc-master.mk index cf8d0edca533..ba51ec163b40 100644 --- a/deploy/iso/minikube-iso/package/runc-master/runc-master.mk +++ b/deploy/iso/minikube-iso/package/runc-master/runc-master.mk @@ -13,15 +13,15 @@ RUNC_MASTER_LICENSE_FILES = LICENSE RUNC_MASTER_DEPENDENCIES = host-go -RUNC_MASTER_GOPATH = "$(@D)/Godeps/_workspace" +RUNC_MASTER_GOPATH = $(@D)/_output RUNC_MASTER_MAKE_ENV = $(HOST_GO_TARGET_ENV) \ CGO_ENABLED=1 \ - GOBIN="$(@D)/bin" \ + GO111MODULE=off \ GOPATH="$(RUNC_MASTER_GOPATH)" \ - PATH=$(BR_PATH) + GOBIN="$(RUNC_MASTER_GOPATH)/bin" \ + PATH=$(RUNC_MASTER_GOPATH)/bin:$(BR_PATH) -RUNC_MASTER_GLDFLAGS = \ - -buildmode=pie -X main.gitCommit=$(RUNC_MASTER_VERSION) +RUNC_MASTER_COMPILE_SRC = $(RUNC_MASTER_GOPATH)/src/github.com/opencontainers/runc ifeq ($(BR2_PACKAGE_LIBSECCOMP),y) RUNC_MASTER_GOTAGS += seccomp @@ -34,13 +34,11 @@ define RUNC_MASTER_CONFIGURE_CMDS endef define RUNC_MASTER_BUILD_CMDS - cd $(@D) && $(RUNC_MASTER_MAKE_ENV) $(HOST_DIR)/usr/bin/go \ - build -v -o $(@D)/bin/runc \ - -tags "$(RUNC_MASTER_GOTAGS)" -ldflags "$(RUNC_MASTER_GLDFLAGS)" github.com/opencontainers/runc + PWD=$(RUNC_MASTER_COMPILE_SRC) $(RUNC_MASTER_MAKE_ENV) $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) BUILDTAGS="$(RUNC_MASTER_GOTAGS)" COMMIT_NO=$(RUNC_MASTER_VERSION) PREFIX=/usr endef define RUNC_MASTER_INSTALL_TARGET_CMDS - $(INSTALL) -D -m 0755 $(@D)/bin/runc $(TARGET_DIR)/usr/bin/runc + $(INSTALL) -D -m 0755 $(@D)/runc $(TARGET_DIR)/usr/bin/runc endef $(eval $(generic-package)) diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile new file mode 100644 index 000000000000..9d4aed5e885f --- /dev/null +++ b/hack/images/kicbase.Dockerfile @@ -0,0 +1,18 @@ +ARG COMMIT_SHA +FROM kindest/node:v1.16.2 +USER root +RUN apt-get update && apt-get install -y \ + sudo \ + dnsutils \ + && apt-get clean -y +RUN rm -rf \ + /var/cache/debconf/* \ + /var/lib/apt/lists/* \ + /var/log/* \ + /tmp/* \ + /var/tmp/* \ + /usr/share/doc/* \ + /usr/share/man/* \ + /usr/share/local/* \ + /kind/bin/kubeadm /kind/bin/kubelet /kind/systemd /kind/images /kind/manifests +RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt" diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index fa84a8c17bd2..fce0f103a22d 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -319,7 +319,9 @@ touch "${JSON_OUT}" echo ">> Running go test2json" go tool test2json -t < "${TEST_OUT}" > "${JSON_OUT}" || true echo ">> Installing gopogh" +cd /tmp GO111MODULE="on" go get -u github.com/medyagh/gopogh@v0.0.17 || true +cd - echo ">> Running gopogh" if test -f "${HTML_OUT}"; then rm "${HTML_OUT}" || true # clean up previous runs of same build diff --git a/hack/jenkins/release_build_and_upload.sh b/hack/jenkins/release_build_and_upload.sh index c217a6a388db..6640fbfd798b 100755 --- a/hack/jenkins/release_build_and_upload.sh +++ b/hack/jenkins/release_build_and_upload.sh @@ -55,7 +55,11 @@ gsutil -m cp out/* "gs://$BUCKET/releases/$TAGNAME/" # Update "latest" release for non-beta/non-alpha builds if ! [[ ${VERSION_BUILD} =~ ^[0-9]+$ ]]; then echo "NOTE: ${VERSION} appears to be a non-standard release, not updating /releases/latest" -else - echo "Updating latest bucket for ${VERSION} release" - gsutil cp -r "gs://${BUCKET}/releases/${TAGNAME}/*" "gs://${BUCKET}/releases/latest/" -fi \ No newline at end of file + exit 0 +fi + +echo "Updating Docker images ..." +make push-gvisor-addon-image push-storage-provisioner-image + +echo "Updating latest bucket for ${VERSION} release ..." +gsutil cp -r "gs://${BUCKET}/releases/${TAGNAME}/*" "gs://${BUCKET}/releases/latest/" diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go new file mode 100644 index 000000000000..51ab37907a02 --- /dev/null +++ b/pkg/addons/addons.go @@ -0,0 +1,233 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addons + +import ( + "fmt" + "os" + "strconv" + + "github.com/pkg/errors" + "github.com/spf13/viper" + "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/cluster" + "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/machine" + "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/storageclass" + pkgutil "k8s.io/minikube/pkg/util" +) + +// defaultStorageClassProvisioner is the name of the default storage class provisioner +const defaultStorageClassProvisioner = "standard" + +// Set sets a value +func Set(name, value, profile string) error { + a, valid := isAddonValid(name) + if !valid { + return errors.Errorf("%s is not a valid addon", name) + } + + // Run any additional validations for this property + if err := run(name, value, profile, a.validations); err != nil { + return errors.Wrap(err, "running validations") + } + + // Set the value + c, err := config.Load(profile) + if err != nil { + return errors.Wrap(err, "loading profile") + } + + if err := a.set(c, name, value); err != nil { + return errors.Wrap(err, "setting new value of addon") + } + + // Run any callbacks for this property + if err := run(name, value, profile, a.callbacks); err != nil { + return errors.Wrap(err, "running callbacks") + } + + // Write the value + return config.Write(profile, c) +} + +// Runs all the validation or callback functions and collects errors +func run(name, value, profile string, fns []setFn) error { + var errors []error + for _, fn := range fns { + err := fn(name, value, profile) + if err != nil { + errors = append(errors, err) + } + } + if len(errors) > 0 { + return fmt.Errorf("%v", errors) + } + return nil +} + +// SetBool sets a bool value +func SetBool(m *config.MachineConfig, name string, val string) error { + b, err := strconv.ParseBool(val) + if err != nil { + return err + } + if m.Addons == nil { + m.Addons = map[string]bool{} + } + m.Addons[name] = b + return nil +} + +// enableOrDisableAddon updates addon status executing any commands necessary +func enableOrDisableAddon(name, val, profile string) error { + enable, err := strconv.ParseBool(val) + if err != nil { + return errors.Wrapf(err, "parsing bool: %s", name) + } + addon := assets.Addons[name] + + // check addon status before enabling/disabling it + alreadySet, err := isAddonAlreadySet(addon, enable) + if err != nil { + out.ErrT(out.Conflict, "{{.error}}", out.V{"error": err}) + return err + } + //if addon is already enabled or disabled, do nothing + if alreadySet { + return nil + } + + if name == "istio" && enable { + minMem := 8192 + minCpus := 4 + memorySizeMB := pkgutil.CalculateSizeInMB(viper.GetString("memory")) + cpuCount := viper.GetInt("cpus") + if memorySizeMB < minMem || cpuCount < minCpus { + out.WarningT("Enable istio needs {{.minMem}} MB of memory and {{.minCpus}} CPUs.", out.V{"minMem": minMem, "minCpus": minCpus}) + } + } + + // TODO(r2d4): config package should not reference API, pull this out + api, err := machine.NewAPIClient() + if err != nil { + return errors.Wrap(err, "machine client") + } + defer api.Close() + + //if minikube is not running, we return and simply update the value in the addon + //config and rewrite the file + if !cluster.IsMinikubeRunning(api) { + return nil + } + + cfg, err := config.Load(profile) + if err != nil && !os.IsNotExist(err) { + exit.WithCodeT(exit.Data, "Unable to load config: {{.error}}", out.V{"error": err}) + } + + host, err := cluster.CheckIfHostExistsAndLoad(api, cfg.Name) + if err != nil { + return errors.Wrap(err, "getting host") + } + + cmd, err := machine.CommandRunner(host) + if err != nil { + return errors.Wrap(err, "command runner") + } + + data := assets.GenerateTemplateData(cfg.KubernetesConfig) + return enableOrDisableAddonInternal(addon, cmd, data, enable) +} + +func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) { + addonStatus, err := addon.IsEnabled() + + if err != nil { + return false, errors.Wrap(err, "get the addon status") + } + + if addonStatus && enable { + return true, nil + } else if !addonStatus && !enable { + return true, nil + } + + return false, nil +} + +func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool) error { + var err error + + updateFile := cmd.Copy + if !enable { + updateFile = cmd.Remove + } + + for _, addon := range addon.Assets { + var addonFile assets.CopyableFile + if addon.IsTemplate() { + addonFile, err = addon.Evaluate(data) + if err != nil { + return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName()) + } + + } else { + addonFile = addon + } + if err := updateFile(addonFile); err != nil { + return errors.Wrapf(err, "updating addon %s", addon.AssetName) + } + } + return nil +} + +// enableOrDisableStorageClasses enables or disables storage classes +func enableOrDisableStorageClasses(name, val, profile string) error { + enable, err := strconv.ParseBool(val) + if err != nil { + return errors.Wrap(err, "Error parsing boolean") + } + + class := defaultStorageClassProvisioner + if name == "storage-provisioner-gluster" { + class = "glusterfile" + } + storagev1, err := storageclass.GetStoragev1() + if err != nil { + return errors.Wrapf(err, "Error getting storagev1 interface %v ", err) + } + + if enable { + // Only StorageClass for 'name' should be marked as default + err = storageclass.SetDefaultStorageClass(storagev1, class) + if err != nil { + return errors.Wrapf(err, "Error making %s the default storage class", class) + } + } else { + // Unset the StorageClass as default + err := storageclass.DisableDefaultStorageClass(storagev1, class) + if err != nil { + return errors.Wrapf(err, "Error disabling %s as the default storage class", class) + } + } + + return enableOrDisableAddon(name, val, profile) +} diff --git a/pkg/addons/addons_test.go b/pkg/addons/addons_test.go new file mode 100644 index 000000000000..2e7a6bed07dd --- /dev/null +++ b/pkg/addons/addons_test.go @@ -0,0 +1,109 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addons + +import ( + "fmt" + "os" + "testing" + + "gotest.tools/assert" + "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/config" +) + +func TestIsAddonAlreadySet(t *testing.T) { + testCases := []struct { + addonName string + }{ + { + addonName: "ingress", + }, + + { + addonName: "registry", + }, + } + + for _, test := range testCases { + addon := assets.Addons[test.addonName] + addonStatus, _ := addon.IsEnabled() + + alreadySet, err := isAddonAlreadySet(addon, addonStatus) + if !alreadySet { + if addonStatus { + t.Errorf("Did not get expected status, \n\n expected %+v already enabled", test.addonName) + } else { + t.Errorf("Did not get expected status, \n\n expected %+v already disabled", test.addonName) + } + } + if err != nil { + t.Errorf("Got unexpected error: %+v", err) + } + } +} + +func TestDisableUnknownAddon(t *testing.T) { + tmpProfile := "temp-minikube-profile" + if err := Set("InvalidAddon", "false", tmpProfile); err == nil { + t.Fatalf("Disable did not return error for unknown addon") + } +} + +func TestEnableUnknownAddon(t *testing.T) { + tmpProfile := "temp-minikube-profile" + if err := Set("InvalidAddon", "true", tmpProfile); err == nil { + t.Fatalf("Enable did not return error for unknown addon") + } +} + +func TestEnableAndDisableAddon(t *testing.T) { + tests := []struct { + name string + enable bool + }{ + { + name: "test enable", + enable: true, + }, { + name: "test disable", + enable: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tmpProfile := "temp-minikube-profile" + if err := os.MkdirAll(config.ProfileFolderPath(tmpProfile), 0777); err != nil { + t.Fatalf("error creating temporary directory") + } + defer os.RemoveAll(config.ProfileFolderPath(tmpProfile)) + + if err := config.DefaultLoader.WriteConfigToFile(tmpProfile, &config.MachineConfig{}); err != nil { + t.Fatalf("error creating temporary profile config: %v", err) + } + if err := Set("dashboard", fmt.Sprintf("%t", test.enable), tmpProfile); err != nil { + t.Fatalf("Disable returned unexpected error: " + err.Error()) + } + c, err := config.DefaultLoader.LoadConfigFromFile(tmpProfile) + if err != nil { + t.Fatalf("error loading config: %v", err) + } + assert.Equal(t, c.Addons["dashboard"], test.enable) + }) + } +} diff --git a/pkg/addons/config.go b/pkg/addons/config.go new file mode 100644 index 000000000000..45d442233229 --- /dev/null +++ b/pkg/addons/config.go @@ -0,0 +1,130 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addons + +import "k8s.io/minikube/pkg/minikube/config" + +type setFn func(string, string, string) error + +// Addon represents an addon +type Addon struct { + name string + set func(*config.MachineConfig, string, string) error + validations []setFn + callbacks []setFn +} + +// Addons is a list of all addons +var Addons = []*Addon{ + { + name: "addon-manager", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "dashboard", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + + { + name: "default-storageclass", + set: SetBool, + callbacks: []setFn{enableOrDisableStorageClasses}, + }, + { + name: "efk", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "freshpod", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "gvisor", + set: SetBool, + validations: []setFn{IsContainerdRuntime}, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "helm-tiller", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "ingress", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "ingress-dns", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "istio-provisioner", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "istio", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "logviewer", + set: SetBool, + }, + { + name: "metrics-server", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "nvidia-driver-installer", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "nvidia-gpu-device-plugin", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + + { + name: "registry", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "registry-creds", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "storage-provisioner", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, + { + name: "storage-provisioner-gluster", + set: SetBool, + callbacks: []setFn{enableOrDisableStorageClasses}, + }, +} diff --git a/pkg/addons/validations.go b/pkg/addons/validations.go new file mode 100644 index 000000000000..53a73822bec4 --- /dev/null +++ b/pkg/addons/validations.go @@ -0,0 +1,62 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addons + +import ( + "fmt" + + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/cruntime" +) + +// containerdOnlyMsg is the message shown when a containerd-only addon is enabled +const containerdOnlyAddonMsg = ` +This addon can only be enabled with the containerd runtime backend. To enable this backend, please first stop minikube with: + +minikube stop + +and then start minikube again with the following flags: + +minikube start --container-runtime=containerd --docker-opt containerd=/var/run/containerd/containerd.sock` + +// IsContainerdRuntime is a validator which returns an error if the current runtime is not containerd +func IsContainerdRuntime(_, _, profile string) error { + config, err := config.Load(profile) + if err != nil { + return fmt.Errorf("config.Load: %v", err) + } + r, err := cruntime.New(cruntime.Config{Type: config.KubernetesConfig.ContainerRuntime}) + if err != nil { + return err + } + _, ok := r.(*cruntime.Containerd) + if !ok { + return fmt.Errorf(containerdOnlyAddonMsg) + } + return nil +} + +// isAddonValid returns the addon, true if it is valid +// otherwise returns nil, false +func isAddonValid(name string) (*Addon, bool) { + for _, a := range Addons { + if a.name == name { + return a, true + } + } + return nil, false +} diff --git a/pkg/addons/validations_test.go b/pkg/addons/validations_test.go new file mode 100644 index 000000000000..23c2441f8f91 --- /dev/null +++ b/pkg/addons/validations_test.go @@ -0,0 +1,45 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addons + +import "testing" + +func TestIsAddonValid(t *testing.T) { + tests := []struct { + description string + name string + isValid bool + }{ + { + description: "valid addon", + name: "gvisor", + isValid: true, + }, { + description: "invalid addon", + name: "invalid", + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + _, valid := isAddonValid(test.name) + if test.isValid != valid { + t.Fatalf("expected: %t\nactual:%t\n", test.isValid, valid) + } + }) + } +} diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index cdc63bde6e39..ef05e3032366 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -29,9 +29,19 @@ import ( "k8s.io/minikube/pkg/drivers/kic/node" "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/minikube/constants" ) -// https://minikube.sigs.k8s.io/docs/reference/drivers/kic/ +// DefaultPodCIDR is The CIDR to be used for pods inside the node. +const DefaultPodCIDR = "10.244.0.0/16" + +// DefaultBindIPV4 is The default IP the container will bind to. +const DefaultBindIPV4 = "127.0.0.1" + +// BaseImage is the base image is used to spin up kic containers +const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.1@sha256:c4ad2938877d2ae0d5b7248a5e7182ff58c0603165c3bedfe9d503e2d380a0db" + +// Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/kic/ type Driver struct { *drivers.BaseDriver *pkgdrivers.CommonDriver @@ -43,16 +53,16 @@ type Driver struct { // Config is configuration for the kic driver used by registry type Config struct { - MachineName string // maps to the container name being created - CPU int // Number of CPU cores assigned to the container - Memory int // max memory in MB - StorePath string // lib machine store path - OCIBinary string // oci tool to use (docker, podman,...) - ImageDigest string // image name with sha to use for the node - APIServerPort int32 // port to connect to forward from container to user's machine - Mounts []oci.Mount // mounts - PortMappings []oci.PortMapping // container port mappings - Envs map[string]string // key,value of environment variables passed to the node + MachineName string // maps to the container name being created + CPU int // Number of CPU cores assigned to the container + Memory int // max memory in MB + StorePath string // libmachine store path + OCIBinary string // oci tool to use (docker, podman,...) + ImageDigest string // image name with sha to use for the node + HostBindPort int // port to connect to forward from container to user's machine + Mounts []oci.Mount // mounts + PortMappings []oci.PortMapping // container port mappings + Envs map[string]string // key,value of environment variables passed to the node } // NewDriver returns a fully configured Kic driver @@ -64,6 +74,7 @@ func NewDriver(c Config) *Driver { }, exec: command.NewKICRunner(c.MachineName, c.OCIBinary), NodeConfig: c, + OCIBinary: c.OCIBinary, } return d } @@ -77,15 +88,15 @@ func (d *Driver) Create() error { CPUs: strconv.Itoa(d.NodeConfig.CPU), Memory: strconv.Itoa(d.NodeConfig.Memory) + "mb", Envs: d.NodeConfig.Envs, - ExtraArgs: []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIServerPort)}, + ExtraArgs: []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.HostBindPort)}, OCIBinary: d.NodeConfig.OCIBinary, } // control plane specific options params.PortMappings = append(params.PortMappings, oci.PortMapping{ ListenAddress: "127.0.0.1", - HostPort: d.NodeConfig.APIServerPort, - ContainerPort: 6443, + HostPort: int32(d.NodeConfig.HostBindPort), + ContainerPort: constants.APIServerPort, }) _, err := node.CreateNode(params) @@ -136,23 +147,20 @@ func (d *Driver) GetState() (state.State, error) { if err != nil { return state.Error, errors.Wrapf(err, "error stop node %s", d.MachineName) } - if o == "running" { + switch o { + case "running": return state.Running, nil - } - if o == "exited" { + case "exited": return state.Stopped, nil - } - if o == "paused" { + case "paused": return state.Paused, nil - } - if o == "restarting" { + case "restarting": return state.Starting, nil - } - if o == "dead" { + case "dead": return state.Error, nil + default: + return state.None, fmt.Errorf("unknown state") } - return state.None, fmt.Errorf("unknown state") - } // Kill stops a host forcefully, including any containers that we are managing. @@ -250,32 +258,3 @@ func (d *Driver) nodeID(nameOrID string) (string, error) { } return string(id), err } - -func ImageForVersion(ver string) (string, error) { - switch ver { - case "v1.11.10": - return "medyagh/kic:v1.11.10@sha256:23bb7f5e8dd2232ec829132172e87f7b9d8de65269630989e7dac1e0fe993b74", nil - case "v1.12.8": - return "medyagh/kic:v1.12.8@sha256:c74bc5f3efe3539f6e1ad7f11bf7c09f3091c0547cb28071f4e43067053e5898", nil - case "v1.12.9": - return "medyagh/kic:v1.12.9@sha256:ff82f58e18dcb22174e8eb09dae14f7edd82d91a83c7ef19e33298d0eba6a0e3", nil - case "v1.12.10": - return "medyagh/kic:v1.12.10@sha256:2d174bae7c20698e59791e7cca9b6db234053d1a92a009d5bb124e482540c70b", nil - case "v1.13.6": - return "medyagh/kic:v1.13.6@sha256:cf63e50f824fe17b90374d38d64c5964eb9fe6b3692669e1201fcf4b29af4964", nil - case "v1.13.7": - return "medyagh/kic:v1.13.7@sha256:1a6a5e1c7534cf3012655e99df680496df9bcf0791a304adb00617d5061233fa", nil - case "v1.14.3": - return "medyagh/kic:v1.14.3@sha256:cebec21f6af23d5dfa3465b88ddf4a1acb94c2c20a0a6ff8cc1c027b0a4e2cec", nil - case "v1.15.0": - return "medyagh/kic:v1.15.0@sha256:40d433d00a2837c8be829bd3cb0576988e377472062490bce0b18281c7f85303", nil - case "v1.15.3": - return "medyagh/kic:v1.15.3@sha256:f05ce52776a86c6ead806942d424de7076af3f115b0999332981a446329e6cf1", nil - case "v1.16.1": - return "medyagh/kic:v1.16.1@sha256:e74530d22e6a04442a97a09bdbba885ad693fcc813a0d1244da32666410d1ad1", nil - case "v1.16.2": - return "medyagh/kic:v1.16.2@sha256:3374a30971bf5b0011441a227fa56ef990b76125b36ca0ab8316a3c7e4f137a3", nil - default: - return "medyagh/kic:v1.16.2@sha256:3374a30971bf5b0011441a227fa56ef990b76125b36ca0ab8316a3c7e4f137a3", nil - } -} diff --git a/pkg/drivers/kic/node/node.go b/pkg/drivers/kic/node/node.go index acb8005a6ea1..6f1aa60c56ed 100644 --- a/pkg/drivers/kic/node/node.go +++ b/pkg/drivers/kic/node/node.go @@ -39,8 +39,8 @@ const ( // Node represents a handle to a kic node // This struct must be created by one of: CreateControlPlane type Node struct { - // must be one of docker container ID or name - name string + id string // container id + name string // container name r command.Runner // Runner ociBinary string } @@ -123,13 +123,15 @@ func CreateNode(p CreateConfig) (*Node, error) { // Find finds a node func Find(ociBinary string, name string, cmder command.Runner) (*Node, error) { - _, err := oci.Inspect(ociBinary, name, "{{.Id}}") + n, err := oci.Inspect(ociBinary, name, "{{.Id}}") if err != nil { return nil, fmt.Errorf("can't find node %v", err) } return &Node{ - name: name, - r: cmder, + ociBinary: ociBinary, + id: n[0], + name: name, + r: cmder, }, nil } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 77235d73ee98..2c4e899055e8 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -53,15 +53,15 @@ func Status(ociBinary string, ociID string) (state.State, error) { out, err := cmd.CombinedOutput() o := strings.Trim(string(out), "\n") s := state.Error - if o == "running" { // TODO: parse all kind of states + switch o { + case "running": s = state.Running - } - if o == "exited" { + case "exited": s = state.Stopped - } - - if o == "paused" { + case "paused": s = state.Paused + case "restaring": + s = state.Starting } if err != nil { @@ -108,7 +108,6 @@ func Pause(ociBinary string, ociID string) error { // Inspect return low-level information on containers func Inspect(ociBinary string, containerNameOrID, format string) ([]string, error) { - cmd := exec.Command(ociBinary, "inspect", "-f", format, containerNameOrID) // ... against the "node" container @@ -350,16 +349,9 @@ func CreateContainer(ociBinary string, image string, opts ...CreateOpt) ([]strin for scanner.Scan() { output = append(output, scanner.Text()) } - // TODO : check for exist status 125 that means it alread exists, we can re-start it - // example error: - // $ docker run --cpus=2 --memory=2000m -d -t --privileged --security-opt seccomp=unconfined --tmpfs /tmp --tmpfs /run -v /lib/modules:/lib/modules:ro --hostname p1control-plane --name p1control-plane --label io.k8s.sigs.kic.clusterp1 --label io.k8s.sigs.kic.role=control-plane --expose 50182 --publish=127.0.0.1:50182:6443 medyagh/kic:v1.15.0@sha256:1f03b3168ffe8ab43ce170a5729e31b0d53fb3a1af88e1ad1bdf4626fad8a91c - // docker: Error response from daemon: Conflict. The container name "/p1control-plane" is already in use by container "0204dcf3ca51c874b6c7dac989beae9d98dd44af53e0a17312f4d3480c1f6191". You have to remove (or rename) that container to be able to reuse that name. - // See 'docker run --help'. - // $ echo $? - // 125 if err != nil { - return output, errors.Wrapf(err, "CreateContainer %v ", args) + return output, errors.Wrapf(err, "args: %v output: %s ", args, output) } return output, nil } diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go index 18b0c73c7fa9..6e35a79dc3d5 100644 --- a/pkg/drivers/kic/oci/types.go +++ b/pkg/drivers/kic/oci/types.go @@ -16,6 +16,11 @@ limitations under the License. package oci +const ( + Docker = "docker" + Podman = "podman" +) + /* These types are from https://github.com/kubernetes/kubernetes/blob/063e7ff358fdc8b0916e6f39beedc0d025734cb1/pkg/kubelet/apis/cri/runtime/v1alpha2/api.pb.go#L183 diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 94d7e9200930..75f4e87f23de 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -22,9 +22,9 @@ import ( "path" "path/filepath" "runtime" - "strconv" "github.com/pkg/errors" + "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/localpath" @@ -54,15 +54,13 @@ func (a *Addon) Name() string { return a.addonName } -// IsEnabled checks if an Addon is enabled +// IsEnabled checks if an Addon is enabled for the current profile func (a *Addon) IsEnabled() (bool, error) { - addonStatusText, err := config.Get(a.addonName) + c, err := config.Load(viper.GetString(config.MachineProfile)) if err == nil { - addonStatus, err := strconv.ParseBool(addonStatusText) - if err != nil { - return false, err + if status, ok := c.Addons[a.Name()]; ok { + return status, nil } - return addonStatus, nil } return a.enabled, nil } diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index 4252d7061758..2ddb642c7554 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -56,22 +56,10 @@ const ( // GetCachedBinaryList returns the list of binaries func GetCachedBinaryList(bootstrapper string) []string { - switch bootstrapper { - case Kubeadm: - return constants.KubeadmBinaries - default: - return []string{} - } + return constants.KubeadmBinaries } // GetCachedImageList returns the list of images for a version func GetCachedImageList(imageRepository string, version string, bootstrapper string) ([]string, error) { - switch bootstrapper { - case Kubeadm: - return images.Kubeadm(imageRepository, version) - case KIC: - return []string{"alpine"}, nil // for testing purpose just caching alpine for kicbs - default: - return []string{}, nil - } + return images.Kubeadm(imageRepository, version) } diff --git a/pkg/minikube/bootstrapper/bsutil/binaries.go b/pkg/minikube/bootstrapper/bsutil/binaries.go index bd59a8e9eb06..2b218a7373bb 100644 --- a/pkg/minikube/bootstrapper/bsutil/binaries.go +++ b/pkg/minikube/bootstrapper/bsutil/binaries.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil package will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( diff --git a/pkg/minikube/bootstrapper/bsutil/extraconfig.go b/pkg/minikube/bootstrapper/bsutil/extraconfig.go index d00c539baf61..0136fea42c90 100644 --- a/pkg/minikube/bootstrapper/bsutil/extraconfig.go +++ b/pkg/minikube/bootstrapper/bsutil/extraconfig.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( diff --git a/pkg/minikube/bootstrapper/bsutil/featuregates.go b/pkg/minikube/bootstrapper/bsutil/featuregates.go index 0f0c49092561..104dbf5a47fb 100644 --- a/pkg/minikube/bootstrapper/bsutil/featuregates.go +++ b/pkg/minikube/bootstrapper/bsutil/featuregates.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( diff --git a/pkg/minikube/bootstrapper/bsutil/featuregates_test.go b/pkg/minikube/bootstrapper/bsutil/featuregates_test.go index 3d3a4fb0c6c8..bdedff7e50cf 100644 --- a/pkg/minikube/bootstrapper/bsutil/featuregates_test.go +++ b/pkg/minikube/bootstrapper/bsutil/featuregates_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( diff --git a/pkg/minikube/bootstrapper/bsutil/files.go b/pkg/minikube/bootstrapper/bsutil/files.go index 7dff9a2dd46f..decb2fed55b8 100644 --- a/pkg/minikube/bootstrapper/bsutil/files.go +++ b/pkg/minikube/bootstrapper/bsutil/files.go @@ -14,12 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( "path" + "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/vmpath" @@ -42,7 +43,6 @@ func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byt assets.NewMemoryAssetTarget(kubeadm, KubeadmYamlPath, "0640"), assets.NewMemoryAssetTarget(kubelet, KubeletSystemdConfFile, "0644"), assets.NewMemoryAssetTarget(kubeletSvc, KubeletServiceFile, "0644"), - assets.NewMemoryAssetTarget(defaultCNIConfig, DefaultCNIConfigPath, "0644"), } // Copy the default CNI config (k8s.conf), so that kubelet can successfully // start a Pod in the case a user hasn't manually installed any CNI plugin @@ -52,3 +52,33 @@ func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byt } return fs } + +// AddAddons adds addons to list of files +func AddAddons(files *[]assets.CopyableFile, data interface{}) error { + // add addons to file list + // custom addons + if err := assets.AddMinikubeDirAssets(files); err != nil { + return errors.Wrap(err, "adding minikube dir assets") + } + // bundled addons + for _, addonBundle := range assets.Addons { + if isEnabled, err := addonBundle.IsEnabled(); err == nil && isEnabled { + for _, addon := range addonBundle.Assets { + if addon.IsTemplate() { + addonFile, err := addon.Evaluate(data) + if err != nil { + return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName()) + } + + *files = append(*files, addonFile) + } else { + *files = append(*files, addon) + } + } + } else if err != nil { + return nil + } + } + + return nil +} diff --git a/pkg/minikube/bootstrapper/bsutil/template/kubelet.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/kubelet.go similarity index 98% rename from pkg/minikube/bootstrapper/bsutil/template/kubelet.go rename to pkg/minikube/bootstrapper/bsutil/ktmpl/kubelet.go index a35bf7d2e1e0..04c7c779c7ae 100644 --- a/pkg/minikube/bootstrapper/bsutil/template/kubelet.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/kubelet.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package template +package ktmpl import "text/template" diff --git a/cmd/minikube/cmd/config/enable_test.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/template.go similarity index 52% rename from cmd/minikube/cmd/config/enable_test.go rename to pkg/minikube/bootstrapper/bsutil/ktmpl/template.go index 28556dc1d8f9..92f0ca3b13cf 100644 --- a/cmd/minikube/cmd/config/enable_test.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/template.go @@ -14,26 +14,28 @@ See the License for the specific language governing permissions and limitations under the License. */ -package config +package ktmpl import ( - "testing" - - "gotest.tools/assert" - pkgConfig "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/localpath" + "fmt" + "sort" ) -func TestEnableUnknownAddon(t *testing.T) { - if err := Set("InvalidAddon", "false"); err == nil { - t.Fatalf("Enable did not return error for unknown addon") +// printMapInOrder sorts the keys and prints the map in order, combining key +// value pairs with the separator character +// +// Note: this is not necessary, but makes testing easy +func printMapInOrder(m map[string]string, sep string) []string { + if m == nil { + return nil } -} - -func TestEnableAddon(t *testing.T) { - if err := Set("ingress", "true"); err != nil { - t.Fatalf("Enable returned unexpected error: " + err.Error()) + keys := []string{} + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + for i, k := range keys { + keys[i] = fmt.Sprintf("%s%s\"%s\"", k, sep, m[k]) } - config, _ := pkgConfig.ReadConfig(localpath.ConfigFile) - assert.Equal(t, config["ingress"], true) + return keys } diff --git a/pkg/minikube/bootstrapper/bsutil/template/template_test.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/template_test.go similarity index 98% rename from pkg/minikube/bootstrapper/bsutil/template/template_test.go rename to pkg/minikube/bootstrapper/bsutil/ktmpl/template_test.go index 4ac1ba3ccc8f..1b3782fdc3e9 100644 --- a/pkg/minikube/bootstrapper/bsutil/template/template_test.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/template_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package template +package ktmpl import ( "reflect" diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha1.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha1.go new file mode 100644 index 000000000000..2576bf07ffb8 --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha1.go @@ -0,0 +1,45 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktmpl + +import "text/template" + +// V1Alpha1 is for Kubernetes v1.11 +var V1Alpha1 = template.Must(template.New("configTmpl-v1alpha1").Funcs(template.FuncMap{ + "printMapInOrder": printMapInOrder, +}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1 +kind: MasterConfiguration +{{if .NoTaintMaster}}noTaintMaster: true{{end}} +api: + advertiseAddress: {{.AdvertiseAddress}} + bindPort: {{.APIServerPort}} + controlPlaneEndpoint: localhost +kubernetesVersion: {{.KubernetesVersion}} +certificatesDir: {{.CertDir}} +networking: + serviceSubnet: {{.ServiceCIDR}} +etcd: + dataDir: {{.EtcdDataDir}} +nodeName: {{.NodeName}} +apiServerCertSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"] +{{if .ImageRepository}}imageRepository: {{.ImageRepository}} +{{end}}{{if .CRISocket}}criSocket: {{.CRISocket}} +{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }} + {{$val}}{{end}} +{{end}}{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}} + {{$i}}: {{$val}}{{end}} +{{end}}`)) diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha3.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha3.go new file mode 100644 index 000000000000..3964e355a1a5 --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha3.go @@ -0,0 +1,69 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktmpl + +import "text/template" + +// V1Alpha3 is for Kubernetes v1.12 +var V1Alpha3 = template.Must(template.New("configTmpl-v1alpha3").Funcs(template.FuncMap{ + "printMapInOrder": printMapInOrder, +}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha3 +kind: InitConfiguration +apiEndpoint: + advertiseAddress: {{.AdvertiseAddress}} + bindPort: {{.APIServerPort}} +bootstrapTokens: + - groups: + - system:bootstrappers:kubeadm:default-node-token + ttl: 24h0m0s + usages: + - signing + - authentication +nodeRegistration: + criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}} + name: {{.NodeName}} + taints: [] +--- +apiVersion: kubeadm.k8s.io/v1alpha3 +kind: ClusterConfiguration +{{if .ImageRepository}}imageRepository: {{.ImageRepository}} +{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }} + {{$val}}{{end}} +{{end -}} +{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}} + {{$i}}: {{$val}}{{end}} +{{end -}} +certificatesDir: {{.CertDir}} +clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"] +controlPlaneEndpoint: localhost:{{.APIServerPort}} +etcd: + local: + dataDir: {{.EtcdDataDir}} +kubernetesVersion: {{.KubernetesVersion}} +networking: + dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} + podSubnet: "{{ .PodSubnet }}" + serviceSubnet: {{.ServiceCIDR}} +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +evictionHard: + nodefs.available: "0%" + nodefs.inodesFree: "0%" + imagefs.available: "0%" +`)) diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go new file mode 100644 index 000000000000..6b6c59a81b4c --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go @@ -0,0 +1,76 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ktmpl + +import "text/template" + +// V1Beta1 is kubeadm config template for Kubernetes v1.13+ +var V1Beta1 = template.Must(template.New("configTmpl-v1beta1").Funcs(template.FuncMap{ + "printMapInOrder": printMapInOrder, +}).Parse(`apiVersion: kubeadm.k8s.io/v1beta1 +kind: InitConfiguration +localAPIEndpoint: + advertiseAddress: {{.AdvertiseAddress}} + bindPort: {{.APIServerPort}} +bootstrapTokens: + - groups: + - system:bootstrappers:kubeadm:default-node-token + ttl: 24h0m0s + usages: + - signing + - authentication +nodeRegistration: + criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}} + name: {{.NodeName}} + taints: [] +--- +apiVersion: kubeadm.k8s.io/v1beta1 +kind: ClusterConfiguration +{{ if .ImageRepository}}imageRepository: {{.ImageRepository}} +{{end}}{{range .ExtraArgs}}{{.Component}}: + extraArgs: +{{- range $i, $val := printMapInOrder .Options ": " }} + {{$val}} +{{- end}} +{{end -}} +{{if .FeatureArgs}}featureGates: +{{range $i, $val := .FeatureArgs}}{{$i}}: {{$val}} +{{end -}}{{end -}} +certificatesDir: {{.CertDir}} +clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"] +controlPlaneEndpoint: localhost:{{.APIServerPort}} +dns: + type: CoreDNS +etcd: + local: + dataDir: {{.EtcdDataDir}} +kubernetesVersion: {{.KubernetesVersion}} +networking: + dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} + podSubnet: "{{.PodSubnet }}" + serviceSubnet: {{.ServiceCIDR}} +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +imageGCHighThresholdPercent: 100 +evictionHard: + nodefs.available: "0%" + nodefs.inodesFree: "0%" + imagefs.available: "0%" +`)) diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go index db580f3622f7..3887909c8317 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go +++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( @@ -24,12 +24,11 @@ import ( "github.com/blang/semver" "github.com/pkg/errors" - "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/template" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/vmpath" - "k8s.io/minikube/pkg/util" ) // Container runtimes @@ -53,7 +52,7 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt return nil, errors.Wrap(err, "generating extra component config for kubeadm") } - // In case of no port assigned, use util.APIServerPort + // In case of no port assigned, use default nodePort := k8s.NodePort if nodePort <= 0 { nodePort = constants.APIServerPort @@ -76,7 +75,7 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt NoTaintMaster bool }{ CertDir: vmpath.GuestCertsDir, - ServiceCIDR: util.DefaultServiceCIDR, + ServiceCIDR: constants.DefaultServiceCIDR, PodSubnet: k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm), AdvertiseAddress: k8s.NodeIP, APIServerPort: nodePort, @@ -97,13 +96,13 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt opts.NoTaintMaster = true b := bytes.Buffer{} - configTmpl := template.KubeAdmConfigTmplV1Alpha1 + configTmpl := ktmpl.V1Alpha1 if version.GTE(semver.MustParse("1.12.0")) { - configTmpl = template.KubeAdmConfigTmplV1Alpha3 + configTmpl = ktmpl.V1Alpha3 } // v1beta1 works in v1.13, but isn't required until v1.14. if version.GTE(semver.MustParse("1.14.0-alpha.0")) { - configTmpl = template.KubeAdmConfigTmplV1Beta1 + configTmpl = ktmpl.V1Beta1 } if err := configTmpl.Execute(&b, opts); err != nil { return nil, err diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go index c0b90a2b4de9..b5fe8c7fe620 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go @@ -101,7 +101,7 @@ This test case has only 1 thing to test and that is the nnetworking/dnsDomain value */ func TestGenerateKubeadmYAMLDNS(t *testing.T) { - versions := []string{"v1.16", "v1.15", "v1.14", "v1.13", "v1.12"} + versions := []string{"v1.17", "v1.16", "v1.15", "v1.14", "v1.13", "v1.12"} tests := []struct { name string runtime string diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go index 48864d80feeb..5c21698deba2 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubelet.go +++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( @@ -22,7 +22,7 @@ import ( "path" "github.com/pkg/errors" - "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/template" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/cruntime" @@ -76,7 +76,7 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, ContainerRuntime: k8s.ContainerRuntime, KubeletPath: path.Join(binRoot(k8s.KubernetesVersion), "kubelet"), } - if err := template.KubeletSystemdTemplate.Execute(&b, opts); err != nil { + if err := ktmpl.KubeletSystemdTemplate.Execute(&b, opts); err != nil { return nil, err } @@ -87,7 +87,7 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, func NewKubeletService(cfg config.KubernetesConfig) ([]byte, error) { var b bytes.Buffer opts := struct{ KubeletPath string }{KubeletPath: path.Join(binRoot(cfg.KubernetesVersion), "kubelet")} - if err := template.KubeletServiceTemplate.Execute(&b, opts); err != nil { + if err := ktmpl.KubeletServiceTemplate.Execute(&b, opts); err != nil { return nil, errors.Wrap(err, "template execute") } return b.Bytes(), nil diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go index 090cab029040..3c93aa97c2d3 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go +++ b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one package bsutil import ( diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go new file mode 100644 index 000000000000..40cb5c46783a --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go @@ -0,0 +1,141 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kverify verifies a running kubernetes cluster is healthy +package kverify + +import ( + "crypto/tls" + "fmt" + "net" + "net/http" + "os/exec" + "time" + + "github.com/docker/machine/libmachine/state" + "github.com/golang/glog" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/minikube/pkg/minikube/command" +) + +// APIServerProcess waits for api server to be healthy returns error if it doesn't +func APIServerProcess(runner command.Runner, start time.Time, timeout time.Duration) error { + glog.Infof("waiting for apiserver process to appear ...") + err := wait.PollImmediate(time.Second*1, timeout, func() (bool, error) { + if time.Since(start) > timeout { + return false, fmt.Errorf("cluster wait timed out during process check") + } + rr, ierr := runner.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver")) + if ierr != nil { + glog.Warningf("pgrep apiserver: %v cmd: %s", ierr, rr.Command()) + return false, nil + } + return true, nil + }) + if err != nil { + return fmt.Errorf("apiserver process never appeared") + } + glog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start)) + return nil +} + +// SystemPods verifies essential pods for running kurnetes is running +func SystemPods(client *kubernetes.Clientset, start time.Time, ip string, port int, timeout time.Duration) error { + glog.Infof("waiting for kube-system pods to appear %s...", net.JoinHostPort(ip, fmt.Sprint(port))) + pStart := time.Now() + podStart := time.Time{} + podList := func() (bool, error) { + if time.Since(start) > timeout { + return false, fmt.Errorf("cluster wait timed out during pod check") + } + // Wait for any system pod, as waiting for apiserver may block until etcd + pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{}) + if len(pods.Items) < 2 { + podStart = time.Time{} + return false, nil + } + if err != nil { + podStart = time.Time{} + return false, nil + } + if podStart.IsZero() { + podStart = time.Now() + } + + glog.Infof("%d kube-system pods found since %s", len(pods.Items), podStart) + if time.Since(podStart) > 2*kconst.APICallRetryInterval { + glog.Infof("stability requirement met, returning") + return true, nil + } + return false, nil + } + if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, podList); err != nil { + return fmt.Errorf("apiserver never returned a pod list") + } + glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart)) + return nil +} + +// APIServerIsRunning waits for api server status to be running +func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Duration) error { + glog.Infof("waiting for apiserver healthz status ...") + hStart := time.Now() + healthz := func() (bool, error) { + if time.Since(start) > timeout { + return false, fmt.Errorf("cluster wait timed out during healthz check") + } + + status, err := APIServerStatus(net.ParseIP(ip), port) + if err != nil { + glog.Warningf("status: %v", err) + return false, nil + } + if status != "Running" { + return false, nil + } + return true, nil + } + + if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, healthz); err != nil { + return fmt.Errorf("apiserver healthz never reported healthy") + } + glog.Infof("duration metric: took %s to wait for apiserver healthz status ...", time.Since(hStart)) + return nil +} + +// APIServerStatus hits the /healthz endpoint and returns libmachine style state.State +func APIServerStatus(ip net.IP, apiserverPort int) (string, error) { + url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(apiserverPort))) + // To avoid: x509: certificate signed by unknown authority + tr := &http.Transport{ + Proxy: nil, // To avoid connectiv issue if http(s)_proxy is set. + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + resp, err := client.Get(url) + // Connection refused, usually. + if err != nil { + return state.Stopped.String(), nil + } + if resp.StatusCode != http.StatusOK { + glog.Warningf("%s response: %v %+v", url, err, resp) + return state.Error.String(), nil + } + return state.Running.String(), nil +} diff --git a/pkg/minikube/bootstrapper/kubeadm/rbac.go b/pkg/minikube/bootstrapper/bsutil/rbac.go similarity index 56% rename from pkg/minikube/bootstrapper/kubeadm/rbac.go rename to pkg/minikube/bootstrapper/bsutil/rbac.go index 728c73163e42..12089bf6ab2d 100644 --- a/pkg/minikube/bootstrapper/kubeadm/rbac.go +++ b/pkg/minikube/bootstrapper/bsutil/rbac.go @@ -14,10 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubeadm +package bsutil import ( + "fmt" "net" + "os/exec" + "strings" "time" "github.com/golang/glog" @@ -25,6 +28,7 @@ import ( rbac "k8s.io/api/rbac/v1beta1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/util/retry" ) @@ -32,9 +36,9 @@ const ( rbacName = "minikube-rbac" ) -// elevateKubeSystemPrivileges gives the kube-system service account +// ElevateKubeSystemPrivileges gives the kube-system service account // cluster admin privileges to work with RBAC. -func elevateKubeSystemPrivileges(client kubernetes.Interface) error { +func ElevateKubeSystemPrivileges(client kubernetes.Interface) error { start := time.Now() clusterRoleBinding := &rbac.ClusterRoleBinding{ ObjectMeta: meta.ObjectMeta{ @@ -67,3 +71,31 @@ func elevateKubeSystemPrivileges(client kubernetes.Interface) error { glog.Infof("duration metric: took %s to wait for elevateKubeSystemPrivileges.", time.Since(start)) return nil } + +// AdjustResourceLimits makes fine adjustments to pod resources that aren't possible via kubeadm config. +func AdjustResourceLimits(c command.Runner) error { + rr, err := c.RunCmd(exec.Command("/bin/bash", "-c", "cat /proc/$(pgrep kube-apiserver)/oom_adj")) + if err != nil { + return errors.Wrapf(err, "oom_adj check cmd %s. ", rr.Command()) + } + glog.Infof("apiserver oom_adj: %s", rr.Stdout.String()) + // oom_adj is already a negative number + if strings.HasPrefix(rr.Stdout.String(), "-") { + return nil + } + glog.Infof("adjusting apiserver oom_adj to -10") + + // Prevent the apiserver from OOM'ing before other pods, as it is our gateway into the cluster. + // It'd be preferable to do this via Kubernetes, but kubeadm doesn't have a way to set pod QoS. + if _, err = c.RunCmd(exec.Command("/bin/bash", "-c", "echo -10 | sudo tee /proc/$(pgrep kube-apiserver)/oom_adj")); err != nil { + return errors.Wrap(err, fmt.Sprintf("oom_adj adjust")) + } + return nil +} + +// ExistingConfig checks if there are config files from possible previous kubernets cluster +func ExistingConfig(c command.Runner) error { + args := append([]string{"ls"}, expectedRemoteArtifacts...) + _, err := c.RunCmd(exec.Command("sudo", args...)) + return err +} diff --git a/pkg/minikube/bootstrapper/bsutil/template/template.go b/pkg/minikube/bootstrapper/bsutil/template/template.go deleted file mode 100644 index 92ac7975aec5..000000000000 --- a/pkg/minikube/bootstrapper/bsutil/template/template.go +++ /dev/null @@ -1,171 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package template - -import ( - "fmt" - "sort" - "text/template" -) - -// KubeAdmConfigTmplV1Alpha1 is for Kubernetes v1.11 -var KubeAdmConfigTmplV1Alpha1 = template.Must(template.New("configTmpl-v1alpha1").Funcs(template.FuncMap{ - "printMapInOrder": printMapInOrder, -}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1 -kind: MasterConfiguration -{{if .NoTaintMaster}}noTaintMaster: true{{end}} -api: - advertiseAddress: {{.AdvertiseAddress}} - bindPort: {{.APIServerPort}} - controlPlaneEndpoint: localhost -kubernetesVersion: {{.KubernetesVersion}} -certificatesDir: {{.CertDir}} -networking: - serviceSubnet: {{.ServiceCIDR}} -etcd: - dataDir: {{.EtcdDataDir}} -nodeName: {{.NodeName}} -{{if .ImageRepository}}imageRepository: {{.ImageRepository}} -{{end}}{{if .CRISocket}}criSocket: {{.CRISocket}} -{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }} - {{$val}}{{end}} -{{end}}{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}} - {{$i}}: {{$val}}{{end}} -{{end}}`)) - -// KubeAdmConfigTmplV1Alpha3 is for Kubernetes v1.12 -var KubeAdmConfigTmplV1Alpha3 = template.Must(template.New("configTmpl-v1alpha3").Funcs(template.FuncMap{ - "printMapInOrder": printMapInOrder, -}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha3 -kind: InitConfiguration -apiEndpoint: - advertiseAddress: {{.AdvertiseAddress}} - bindPort: {{.APIServerPort}} -bootstrapTokens: - - groups: - - system:bootstrappers:kubeadm:default-node-token - ttl: 24h0m0s - usages: - - signing - - authentication -nodeRegistration: - criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}} - name: {{.NodeName}} - taints: [] ---- -apiVersion: kubeadm.k8s.io/v1alpha3 -kind: ClusterConfiguration -{{if .ImageRepository}}imageRepository: {{.ImageRepository}} -{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }} - {{$val}}{{end}} -{{end -}} -{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}} - {{$i}}: {{$val}}{{end}} -{{end -}} -certificatesDir: {{.CertDir}} -clusterName: kubernetes -controlPlaneEndpoint: localhost:{{.APIServerPort}} -etcd: - local: - dataDir: {{.EtcdDataDir}} -kubernetesVersion: {{.KubernetesVersion}} -networking: - dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} - podSubnet: {{if .PodSubnet}}{{.PodSubnet}}{{else}}""{{end}} - serviceSubnet: {{.ServiceCIDR}} ---- -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -evictionHard: - nodefs.available: "0%" - nodefs.inodesFree: "0%" - imagefs.available: "0%" -`)) - -// KubeAdmConfigTmplV1Beta1 is for Kubernetes v1.13+ -var KubeAdmConfigTmplV1Beta1 = template.Must(template.New("configTmpl-v1beta1").Funcs(template.FuncMap{ - "printMapInOrder": printMapInOrder, -}).Parse(`apiVersion: kubeadm.k8s.io/v1beta1 -kind: InitConfiguration -localAPIEndpoint: - advertiseAddress: {{.AdvertiseAddress}} - bindPort: {{.APIServerPort}} -bootstrapTokens: - - groups: - - system:bootstrappers:kubeadm:default-node-token - ttl: 24h0m0s - usages: - - signing - - authentication -nodeRegistration: - criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}} - name: {{.NodeName}} - taints: [] ---- -apiVersion: kubeadm.k8s.io/v1beta1 -kind: ClusterConfiguration -{{ if .ImageRepository}}imageRepository: {{.ImageRepository}} -{{end}}{{range .ExtraArgs}}{{.Component}}: - extraArgs: -{{- range $i, $val := printMapInOrder .Options ": " }} - {{$val}} -{{- end}} -{{end -}} -{{if .FeatureArgs}}featureGates: -{{range $i, $val := .FeatureArgs}}{{$i}}: {{$val}} -{{end -}}{{end -}} -certificatesDir: {{.CertDir}} -clusterName: kubernetes -controlPlaneEndpoint: localhost:{{.APIServerPort}} -dns: - type: CoreDNS -etcd: - local: - dataDir: {{.EtcdDataDir}} -kubernetesVersion: {{.KubernetesVersion}} -networking: - dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} - podSubnet: "" - serviceSubnet: {{.ServiceCIDR}} ---- -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -imageGCHighThresholdPercent: 100 -evictionHard: - nodefs.available: "0%" - nodefs.inodesFree: "0%" - imagefs.available: "0%" -`)) - -// printMapInOrder sorts the keys and prints the map in order, combining key -// value pairs with the separator character -// -// Note: this is not necessary, but makes testing easy -func printMapInOrder(m map[string]string, sep string) []string { - if m == nil { - return nil - } - keys := []string{} - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - for i, k := range keys { - keys[i] = fmt.Sprintf("%s%s\"%s\"", k, sep, m[k]) - } - return keys -} diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml index cb5e0e62e5a5..e44ab4c92aba 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] criSocket: /run/containerd/containerd.sock apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml index a78261c550ed..ed91117f7308 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] criSocket: /run/containerd/containerd.sock apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml index a78261c550ed..ed91117f7308 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] criSocket: /run/containerd/containerd.sock apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml index 15bb16aaa158..c15e61950852 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] criSocket: /var/run/crio/crio.sock apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml index 7e33c3c86198..eb4fd3fd1cfe 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] criSocket: /var/run/crio/crio.sock apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml index 13ea3d1c53d9..8e3e566bc215 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml @@ -12,5 +12,6 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml index 5e121b085d19..6e0860426320 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] imageRepository: test/repo apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml index aac8924cbe9b..a3d5c8230327 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml @@ -12,6 +12,7 @@ networking: etcd: dataDir: /var/lib/minikube/etcd nodeName: mk +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" fail-no-swap: "true" diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml index 2c7ad8fc47c6..fee7f3684f18 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:12345 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml index c13a251804da..c3399153f923 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: @@ -28,7 +29,7 @@ etcd: kubernetesVersion: v1.12.0 networking: dnsDomain: cluster.local - podSubnet: 192.168.32.0/20 + podSubnet: "192.168.32.0/20" serviceSubnet: 10.96.0.0/12 --- apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml index 1398527e91fe..9beec13a4d53 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml index d33f0e13dc62..88253bcb486d 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml @@ -29,6 +29,7 @@ schedulerExtraArgs: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml index c076b8ad9f59..4a3555c9afe5 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml index 36856509b97e..7ccd2edf2854 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml index a4a813404c50..96bf607d6551 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml index dab9a37c2353..f0b48f62e13d 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml @@ -22,6 +22,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml index 3a6ff1aeed2e..22809c5ce969 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml @@ -26,6 +26,7 @@ schedulerExtraArgs: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml index f365193c6170..3106e329d1f6 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:12345 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml index d5f799e56fb9..279a7c20a381 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: @@ -28,7 +29,7 @@ etcd: kubernetesVersion: v1.13.0 networking: dnsDomain: cluster.local - podSubnet: 192.168.32.0/20 + podSubnet: "192.168.32.0/20" serviceSubnet: 10.96.0.0/12 --- apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml index 806106b3b7d1..6994f8d92b8a 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml index e602810b0d49..1304deaa3e5b 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml @@ -29,6 +29,7 @@ schedulerExtraArgs: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml index b82b3bf69bb4..a333a2539986 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml index 9995a46c1417..afc4d8a8215b 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml index cdf2a0405fe0..89d56b7bfddc 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml @@ -21,6 +21,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml index b2b8b6e3fb9d..3fc3446ce482 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml @@ -22,6 +22,7 @@ apiServerExtraArgs: enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml index 484dd5a352b4..9a4466840e80 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml @@ -26,6 +26,7 @@ schedulerExtraArgs: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 etcd: local: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml index 5755f5641d6b..6d4eeeeba358 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:12345 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml index f4ff177f6757..3c6997ddecb0 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS @@ -31,7 +33,7 @@ etcd: kubernetesVersion: v1.14.0 networking: dnsDomain: cluster.local - podSubnet: "" + podSubnet: "192.168.32.0/20" serviceSubnet: 10.96.0.0/12 --- apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml index f4ff177f6757..d96e27170cbb 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml index 6f46996151b4..1ad22d184b96 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml @@ -32,6 +32,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml index 47cfd0ecbb79..062233c224e8 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml index 57371bf704a4..bedb175306a9 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml index 93ce010c2ac4..dc8df12a4440 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml index 2c1a895bca58..c9c6846a3813 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml @@ -23,6 +23,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml index 325265c2c6a3..2d0c7e807a94 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml @@ -29,6 +29,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml index 9f08022e331d..30bb67574a6b 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:12345 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml index 30e3b200f136..4fc81aa1c51b 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS @@ -31,7 +33,7 @@ etcd: kubernetesVersion: v1.15.0 networking: dnsDomain: cluster.local - podSubnet: "" + podSubnet: "192.168.32.0/20" serviceSubnet: 10.96.0.0/12 --- apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml index 30e3b200f136..5fcd575dee90 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml index bcc3e64b0030..1d82fb284d6d 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml @@ -32,6 +32,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml index 7cc895ca1583..eadb7a3fcdb2 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml index f31d94301338..5e9b1e4bfe00 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml index d6474d3bee3f..e8e9158ad2e5 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml index aec5820a00a3..1362db1f37c0 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml @@ -23,6 +23,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml index eb354d47ee50..31462f323fa2 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml @@ -29,6 +29,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml index d7e9c9e97596..1befae1db7b1 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:12345 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml index d4de91b7eb07..65ead231372c 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS @@ -31,7 +33,7 @@ etcd: kubernetesVersion: v1.16.0 networking: dnsDomain: cluster.local - podSubnet: "" + podSubnet: "192.168.32.0/20" serviceSubnet: 10.96.0.0/12 --- apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml index d4de91b7eb07..3b269defb341 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml index 5aa8ca8652ac..b04f318a0866 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml @@ -32,6 +32,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml index 57ac32b75eba..c64f2a1cba2f 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml index 4e1949e0f26f..60d7ae8eea10 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml index 1fd0562a5590..08c7528ce35a 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml index f1b08545b9b2..62bf5372ac57 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml @@ -23,6 +23,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml index 6c02b9c29e2d..1c0d76eba4d5 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml @@ -29,6 +29,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml index 166ebbbbe8e6..872fa683e0c8 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:12345 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml index 6bb42a82e8e6..7336030107d8 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS @@ -31,7 +33,7 @@ etcd: kubernetesVersion: v1.17.0 networking: dnsDomain: cluster.local - podSubnet: "" + podSubnet: "192.168.32.0/20" serviceSubnet: 10.96.0.0/12 --- apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml index 6bb42a82e8e6..1f5fb46836cd 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml index 6cc338fb211f..fe876ff8884d 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml @@ -32,6 +32,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml index 0fddd573cb28..6704dc431b93 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml index 53cbaba48837..263492d5bece 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml index e32a188927f3..7441511cb0ff 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml @@ -22,6 +22,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml index 1dd863df2843..c746585e4dd1 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml @@ -23,6 +23,8 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml index 03b1bb3c7205..d47790e95ea0 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml @@ -29,6 +29,8 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: kubernetes +apiServer: + certSANs: ["127.0.0.1", "localhost", "1.1.1.1"] controlPlaneEndpoint: localhost:8443 dns: type: CoreDNS diff --git a/pkg/minikube/bootstrapper/bsutil/types.go b/pkg/minikube/bootstrapper/bsutil/types.go new file mode 100644 index 000000000000..a436fdf85c0b --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/types.go @@ -0,0 +1,29 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one +package bsutil + +// SkipAdditionalPreflights are additional preflights we skip depending on the runtime in use. +var SkipAdditionalPreflights = map[string][]string{} + +// expectedRemoteArtifacts remote artifacts that must exist for minikube to function properly. The sign of a previously working installation. +// NOTE: /etc is not persistent across restarts, so don't bother checking there +var expectedRemoteArtifacts = []string{ + "/var/lib/kubelet/kubeadm-flags.env", + "/var/lib/kubelet/config.yaml", + EtcdDataDir(), +} diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index d0731b24cd01..21c573454cfa 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/clientcmd/api" "k8s.io/client-go/tools/clientcmd/api/latest" + "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" @@ -110,7 +111,7 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error { kcs := &kubeconfig.Settings{ ClusterName: k8s.NodeName, - ClusterServerAddress: fmt.Sprintf("https://localhost:%d", k8s.NodePort), + ClusterServerAddress: fmt.Sprintf("https://%s", net.JoinHostPort("localhost", fmt.Sprint(k8s.NodePort))), ClientCertificate: path.Join(vmpath.GuestCertsDir, "apiserver.crt"), ClientKey: path.Join(vmpath.GuestCertsDir, "apiserver.key"), CertificateAuthority: path.Join(vmpath.GuestCertsDir, "ca.crt"), @@ -175,7 +176,7 @@ func generateCerts(k8s config.KubernetesConfig) error { apiServerIPs := append( k8s.APIServerIPs, - []net.IP{net.ParseIP(k8s.NodeIP), serviceIP, net.ParseIP("10.0.0.1")}...) + []net.IP{net.ParseIP(k8s.NodeIP), serviceIP, net.ParseIP(kic.DefaultBindIPV4), net.ParseIP("10.0.0.1")}...) apiServerNames := append(k8s.APIServerNames, k8s.APIServerName) apiServerAlternateNames := append( apiServerNames, diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go index 7d65e6c3af38..167cf4084e62 100644 --- a/pkg/minikube/bootstrapper/certs_test.go +++ b/pkg/minikube/bootstrapper/certs_test.go @@ -38,7 +38,7 @@ func TestSetupCerts(t *testing.T) { k8s := config.KubernetesConfig{ APIServerName: constants.APIServerName, DNSDomain: constants.ClusterDNSDomain, - ServiceCIDR: util.DefaultServiceCIDR, + ServiceCIDR: constants.DefaultServiceCIDR, } if err := os.Mkdir(filepath.Join(tempDir, "certs"), 0777); err != nil { diff --git a/pkg/minikube/bootstrapper/images/images.go b/pkg/minikube/bootstrapper/images/images.go index b80e17f372c0..80a06f385e7b 100644 --- a/pkg/minikube/bootstrapper/images/images.go +++ b/pkg/minikube/bootstrapper/images/images.go @@ -18,20 +18,99 @@ limitations under the License. package images import ( + "fmt" "path" "runtime" + + "github.com/blang/semver" ) -// ArchTag returns a CPU architecture suffix for images -func ArchTag(hasTag bool) string { +// Pause returns the image name to pull for the pause image +func Pause(mirror string) string { + // Should match `PauseVersion` in: + // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go + return path.Join(kubernetesRepo(mirror), "pause"+archTag(false)+"3.1") +} + +// essentials returns images needed too bootstrap a kubenretes +func essentials(mirror string, v semver.Version) []string { + imgs := []string{ + componentImage("kube-proxy", v, mirror), + componentImage("kube-scheduler", v, mirror), + componentImage("kube-controller-manager", v, mirror), + componentImage("kube-apiserver", v, mirror), + coreDNS(v, mirror), + etcd(v, mirror), + Pause(mirror), + } + return imgs +} + +// componentImage returns a Kubernetes component image to pull +func componentImage(name string, v semver.Version, mirror string) string { + needsArchSuffix := false + ancient := semver.MustParseRange("<1.12.0") + if ancient(v) { + needsArchSuffix = true + } + + return fmt.Sprintf("%sv%s", path.Join(kubernetesRepo(mirror), name+archTag(needsArchSuffix)), v) +} + +// coreDNS returns the images used for CoreDNS +func coreDNS(v semver.Version, mirror string) string { + // Should match `CoreDNSVersion` in + // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go + cv := "1.6.5" + switch v.Minor { + case 16: + cv = "1.6.2" + case 15, 14: + cv = "1.3.1" + case 13: + cv = "1.2.6" + case 12: + cv = "1.2.2" + case 11: + cv = "1.1.3" + } + return path.Join(kubernetesRepo(mirror), "coredns"+archTag(false)+cv) +} + +// etcd returns the image used for etcd +func etcd(v semver.Version, mirror string) string { + needsArchSuffix := false + ancient := semver.MustParseRange("<1.12.0") + if ancient(v) { + needsArchSuffix = true + } + + // Should match `DefaultEtcdVersion` in: + // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go + ev := "3.4.3-0" + switch v.Minor { + case 16: + ev = "3.3.15-0" + case 14, 15: + ev = "3.3.10" + case 12, 13: + ev = "3.2.24" + case 11: + ev = "3.2.18" + } + return path.Join(kubernetesRepo(mirror), "etcd"+archTag(needsArchSuffix)+ev) +} + +// archTag returns a CPU architecture suffix for images +func archTag(hasTag bool) string { if runtime.GOARCH == "amd64" && !hasTag { return ":" } return "-" + runtime.GOARCH + ":" } -// Auxiliary returns images that are helpful for running minikube -func Auxiliary(mirror string) []string { +// auxiliary returns images that are helpful for running minikube +func auxiliary(mirror string) []string { return []string{ addonManager(mirror), storageProvisioner(mirror), @@ -40,21 +119,14 @@ func Auxiliary(mirror string) []string { } } -// Pause returns the image name to pull for the pause image -func Pause(mirror string) string { - // Should match `PauseVersion` in: - // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go - return path.Join(KubernetesRepo(mirror), "pause"+ArchTag(false)+"3.1") -} - // storageProvisioner returns the minikube storage provisioner image func storageProvisioner(mirror string) string { - return path.Join(minikubeRepo(mirror), "storage-provisioner"+ArchTag(false)+"v1.8.1") + return path.Join(minikubeRepo(mirror), "storage-provisioner"+archTag(false)+"v1.8.1") } // addonManager returns the Kubernetes addon manager image func addonManager(mirror string) string { - return path.Join(KubernetesRepo(mirror), "kube-addon-manager"+ArchTag(false)+"v9.0.2") + return path.Join(kubernetesRepo(mirror), "kube-addon-manager"+archTag(false)+"v9.0.2") } // dashboardFrontend returns the image used for the dashboard frontend diff --git a/pkg/minikube/bootstrapper/images/images_test.go b/pkg/minikube/bootstrapper/images/images_test.go index 31a487e4c4f0..3d12adc59131 100644 --- a/pkg/minikube/bootstrapper/images/images_test.go +++ b/pkg/minikube/bootstrapper/images/images_test.go @@ -29,7 +29,7 @@ func TestAuxiliary(t *testing.T) { "kubernetesui/dashboard:v2.0.0-beta8", "kubernetesui/metrics-scraper:v1.0.2", } - got := Auxiliary("") + got := auxiliary("") if diff := cmp.Diff(want, got); diff != "" { t.Errorf("images mismatch (-want +got):\n%s", diff) } @@ -42,7 +42,7 @@ func TestAuxiliaryMirror(t *testing.T) { "test.mirror/dashboard:v2.0.0-beta8", "test.mirror/metrics-scraper:v1.0.2", } - got := Auxiliary("test.mirror") + got := auxiliary("test.mirror") if diff := cmp.Diff(want, got); diff != "" { t.Errorf("images mismatch (-want +got):\n%s", diff) } diff --git a/pkg/minikube/bootstrapper/images/kubeadm.go b/pkg/minikube/bootstrapper/images/kubeadm.go index 7f1071a3f30f..2c3ea23b626c 100644 --- a/pkg/minikube/bootstrapper/images/kubeadm.go +++ b/pkg/minikube/bootstrapper/images/kubeadm.go @@ -17,8 +17,6 @@ limitations under the License. package images import ( - "fmt" - "path" "strings" "github.com/blang/semver" @@ -31,71 +29,7 @@ func Kubeadm(mirror string, version string) ([]string, error) { if err != nil { return nil, errors.Wrap(err, "semver") } - - imgs := []string{ - componentImage("kube-proxy", v, mirror), - componentImage("kube-scheduler", v, mirror), - componentImage("kube-controller-manager", v, mirror), - componentImage("kube-apiserver", v, mirror), - coreDNS(v, mirror), - etcd(v, mirror), - Pause(mirror), - } - imgs = append(imgs, Auxiliary(mirror)...) + imgs := essentials(mirror, v) + imgs = append(imgs, auxiliary(mirror)...) return imgs, nil } - -// componentImage returns a Kubernetes component image to pull -func componentImage(name string, v semver.Version, mirror string) string { - needsArchSuffix := false - ancient := semver.MustParseRange("<1.12.0") - if ancient(v) { - needsArchSuffix = true - } - - return fmt.Sprintf("%sv%s", path.Join(KubernetesRepo(mirror), name+ArchTag(needsArchSuffix)), v) -} - -// coreDNS returns the images used for CoreDNS -func coreDNS(v semver.Version, mirror string) string { - // Should match `CoreDNSVersion` in - // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go - cv := "1.6.5" - switch v.Minor { - case 16: - cv = "1.6.2" - case 15, 14: - cv = "1.3.1" - case 13: - cv = "1.2.6" - case 12: - cv = "1.2.2" - case 11: - cv = "1.1.3" - } - return path.Join(KubernetesRepo(mirror), "coredns"+ArchTag(false)+cv) -} - -// etcd returns the image used for etcd -func etcd(v semver.Version, mirror string) string { - needsArchSuffix := false - ancient := semver.MustParseRange("<1.12.0") - if ancient(v) { - needsArchSuffix = true - } - - // Should match `DefaultEtcdVersion` in: - // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go - ev := "3.4.3-0" - switch v.Minor { - case 16: - ev = "3.3.15-0" - case 14, 15: - ev = "3.3.10" - case 12, 13: - ev = "3.2.24" - case 11: - ev = "3.2.18" - } - return path.Join(KubernetesRepo(mirror), "etcd"+ArchTag(needsArchSuffix)+ev) -} diff --git a/pkg/minikube/bootstrapper/images/repo.go b/pkg/minikube/bootstrapper/images/repo.go index 02cf44345b2f..ca9e3c366a86 100644 --- a/pkg/minikube/bootstrapper/images/repo.go +++ b/pkg/minikube/bootstrapper/images/repo.go @@ -19,8 +19,8 @@ package images // DefaultKubernetesRepo is the default Kubernetes repository const DefaultKubernetesRepo = "k8s.gcr.io" -// KubernetesRepo returns the official Kubernetes repository, or an alternate -func KubernetesRepo(mirror string) string { +// kubernetesRepo returns the official Kubernetes repository, or an alternate +func kubernetesRepo(mirror string) string { if mirror != "" { return mirror } diff --git a/pkg/minikube/bootstrapper/kicbs/cni.go b/pkg/minikube/bootstrapper/kicbs/cni.go index 44eb980ac7cd..caa4534b47d6 100644 --- a/pkg/minikube/bootstrapper/kicbs/cni.go +++ b/pkg/minikube/bootstrapper/kicbs/cni.go @@ -14,12 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bootstrapper for kic +// Package kicbs bootstrapper for kic package kicbs const defaultCNIManifest = ` -# kindnetd networking manifest -# would you kindly template this file --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -99,7 +97,7 @@ spec: fieldRef: fieldPath: status.podIP - name: POD_SUBNET - value: {{ .PodSubnet }} + value: 10.244.0.0/16 volumeMounts: - name: cni-cfg mountPath: /etc/cni/net.d diff --git a/pkg/minikube/bootstrapper/kicbs/kicbs.go b/pkg/minikube/bootstrapper/kicbs/kicbs.go index c613d35e6823..3c7d2ddd6ad7 100644 --- a/pkg/minikube/bootstrapper/kicbs/kicbs.go +++ b/pkg/minikube/bootstrapper/kicbs/kicbs.go @@ -14,21 +14,29 @@ See the License for the specific language governing permissions and limitations under the License. */ -// bootstrapper for kic +// Package kicbs bootstrapper for kic package kicbs import ( "fmt" "net" "os/exec" + "strings" "time" + "github.com/blang/semver" "github.com/docker/machine/libmachine" "github.com/golang/glog" "github.com/pkg/errors" "github.com/spf13/viper" + "k8s.io/client-go/kubernetes" + kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/minikube/pkg/drivers/kic" + "k8s.io/minikube/pkg/kapi" + "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" @@ -36,11 +44,13 @@ import ( "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/vmpath" ) // Bootstrapper is a bootstrapper using kicbs type Bootstrapper struct { c command.Runner + k8sClient *kubernetes.Clientset // kubernetes client used to verify pods inside cluster contextName string } @@ -62,7 +72,7 @@ func NewBootstrapper(api libmachine.API) (*Bootstrapper, error) { func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { images, err := images.Kubeadm(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion) if err != nil { - return errors.Wrap(err, "kubeadm images") + return errors.Wrap(err, "kic images") } if cfg.KubernetesConfig.ShouldLoadCachedImages { @@ -101,15 +111,14 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { return errors.Wrap(err, "downloading binaries") } - var cniFile []byte = nil - if cfg.KubernetesConfig.EnableDefaultCNI { - cniFile = []byte(defaultCNIManifest) - } + cniFile := []byte(defaultCNIManifest) + files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile) - // if err := addAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil { - // return errors.Wrap(err, "adding addons") - // } + if err := bsutil.AddAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil { + return errors.Wrap(err, "adding addons") + } + for _, f := range files { if err := k.c.Copy(f); err != nil { return errors.Wrapf(err, "copy") @@ -122,28 +131,216 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { return nil } -func (k *Bootstrapper) PullImages(config.KubernetesConfig) error { - return fmt.Errorf("the PullImages is not implemented in kicbs yet") +// SetupCerts generates the certs the cluster +func (k *Bootstrapper) SetupCerts(cfg config.KubernetesConfig) error { + return bootstrapper.SetupCerts(k.c, cfg) +} + +// PullImages downloads images that will be used by Kubernetes +func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error { + version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) + if err != nil { + return errors.Wrap(err, "parsing kubernetes version") + } + if version.LT(semver.MustParse("1.11.0")) { + return fmt.Errorf("pull command is not supported by kubeadm v%s", version) + } + + rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s config images pull --config %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath))) + if err != nil { + return errors.Wrapf(err, "running cmd: %q", rr.Command()) + } + return nil +} + +// StartCluster starts the cluster +func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { + k8s.NodeIP = kic.DefaultBindIPV4 + err := bsutil.ExistingConfig(k.c) + if err == nil { // if there is an existing cluster don't reconfigure it + return k.restartCluster(k8s) + } + glog.Infof("existence check: %v", err) + + start := time.Now() + glog.Infof("StartCluster: %+v", k8s) + defer func() { + glog.Infof("StartCluster complete in %s", time.Since(start)) + }() + + extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions) + r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime}) + if err != nil { + return err + } + + ignore := []string{ + fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestManifestsDir, "/", "-", -1)), + fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestPersistentDir, "/", "-", -1)), + fmt.Sprintf("DirAvailable-%s", strings.Replace(bsutil.EtcdDataDir(), "/", "-", -1)), + "FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml", + "FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml", + "FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml", + "FileAvailable--etc-kubernetes-manifests-etcd.yaml", + "FileContent--proc-sys-net-bridge-bridge-nf-call-iptables", // for kic only + "Port-10250", // For "none" users who already have a kubelet online + "Swap", // For "none" users who have swap configured + "SystemVerification", // For kic on linux example error: "modprobe: FATAL: Module configs not found in directory /lib/modules/5.2.17-1rodete3-amd64" + } + ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...) + + c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ","))) + glog.Infof("starting kubeadm init") + if rr, err := k.c.RunCmd(c); err != nil { + return errors.Wrapf(err, "init failed. cmd: %q output: %q", rr.Command(), rr.Output()) + } + + glog.Infof("applying kic overlay network") + if err := k.applyOverlayNetwork(); err != nil { + return errors.Wrap(err, "applying kic overlay network") + } + + glog.Infof("Skipping Configuring cluster permissions for kic...") + + if err := bsutil.AdjustResourceLimits(k.c); err != nil { + glog.Warningf("unable to adjust resource limits: %v", err) + } + + return nil +} + +// restartCluster restarts the Kubernetes cluster configured by kubeadm +func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { + glog.Infof("restartCluster start") + + start := time.Now() + defer func() { + glog.Infof("restartCluster took %s", time.Since(start)) + }() + + version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) + if err != nil { + return errors.Wrap(err, "parsing kubernetes version") + } + + phase := "alpha" + controlPlane := "controlplane" + if version.GTE(semver.MustParse("1.13.0")) { + phase = "init" + controlPlane = "control-plane" + } + + baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), phase) + cmds := []string{ + fmt.Sprintf("%s phase certs all --config %s", baseCmd, bsutil.KubeadmYamlPath), + fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, bsutil.KubeadmYamlPath), + fmt.Sprintf("%s phase %s all --config %s", baseCmd, controlPlane, bsutil.KubeadmYamlPath), + fmt.Sprintf("%s phase etcd local --config %s", baseCmd, bsutil.KubeadmYamlPath), + } + + // Run commands one at a time so that it is easier to root cause failures. + for _, c := range cmds { + rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", c)) + if err != nil { + return errors.Wrapf(err, "running cmd: %s", rr.Command()) + } + } + + // We must ensure that the apiserver is healthy before proceeding + if err := kverify.APIServerProcess(k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil { + return errors.Wrap(err, "apiserver healthz") + } + + client, err := k.client(k8s) + if err != nil { + return errors.Wrap(err, "getting k8s client") + } + + if err := kverify.SystemPods(client, time.Now(), k8s.NodeIP, k8s.NodePort, kconst.DefaultControlPlaneTimeout); err != nil { + return errors.Wrap(err, "system pods") + } + + // Explicitly re-enable kubeadm addons (proxy, coredns) so that they will check for IP or configuration changes. + if rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s phase addon all --config %s", baseCmd, bsutil.KubeadmYamlPath))); err != nil { + return errors.Wrapf(err, fmt.Sprintf("addon phase cmd:%q", rr.Command())) + } + + if err := bsutil.AdjustResourceLimits(k.c); err != nil { + glog.Warningf("unable to adjust resource limits: %v", err) + } + return nil } -func (k *Bootstrapper) StartCluster(config.KubernetesConfig) error { - return fmt.Errorf("the StartCluster is not implemented in kicbs yet") + +// WaitForCluster blocks until the cluster appears to be healthy +func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time.Duration) error { + start := time.Now() + out.T(out.Waiting, "Waiting for cluster to come online ...") + if err := kverify.APIServerProcess(k.c, start, timeout); err != nil { + return errors.Wrap(err, "wait for api proc") + } + + if err := kverify.APIServerIsRunning(start, "127.0.0.1", k8s.NodePort, timeout); err != nil { + return err + } + + c, err := k.client(k8s) // getting kubernetes client before polling. + if err != nil { + return errors.Wrap(err, "get k8s client") + } + + if err := kverify.SystemPods(c, start, "127.0.0.1", k8s.NodePort, timeout); err != nil { + return errors.Wrap(err, "wait for system pods") + } + + return nil } func (k *Bootstrapper) DeleteCluster(config.KubernetesConfig) error { return fmt.Errorf("the DeleteCluster is not implemented in kicbs yet") } -func (k *Bootstrapper) WaitForCluster(config.KubernetesConfig, time.Duration) error { - return fmt.Errorf("the WaitForCluster is not implemented in kicbs yet") -} + func (k *Bootstrapper) LogCommands(bootstrapper.LogOptions) map[string]string { return map[string]string{} } -func (k *Bootstrapper) SetupCerts(cfg config.KubernetesConfig) error { - return fmt.Errorf("the SetupCerts is not implemented in kicbs yet") -} + func (k *Bootstrapper) GetKubeletStatus() (string, error) { return "", fmt.Errorf("the GetKubeletStatus is not implemented in kicbs yet") } func (k *Bootstrapper) GetAPIServerStatus(net.IP, int) (string, error) { return "", fmt.Errorf("the GetAPIServerStatus is not implemented in kicbs yet") } + +// client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver +func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) { + if k.k8sClient != nil { + return k.k8sClient, nil + } + + config, err := kapi.ClientConfig(k.contextName) + if err != nil { + return nil, errors.Wrap(err, "client config") + } + + endpoint := fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(k8s.NodePort))) + if config.Host != endpoint { + glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint) + config.Host = endpoint + } + c, err := kubernetes.NewForConfig(config) + if err == nil { + k.k8sClient = c + } + return c, err +} + +// applyOverlayNetwork applies the CNI plugin needed to make kic work +func (k *Bootstrapper) applyOverlayNetwork() error { + cmd := exec.Command( + "kubectl", "create", "--kubeconfig=/etc/kubernetes/admin.conf", + "-f", bsutil.DefaultCNIConfigPath, + ) + if rr, err := k.c.RunCmd(cmd); err != nil { + return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output()) + } + return nil +} diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 1b8458651616..f5cfe92dc367 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -17,15 +17,14 @@ limitations under the License. package kubeadm import ( - "crypto/tls" "os/exec" "fmt" "net" - "net/http" // WARNING: Do not use path/filepath in this package unless you want bizarre Windows paths + "strconv" "strings" "time" @@ -35,14 +34,13 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" "github.com/spf13/viper" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" @@ -54,20 +52,10 @@ import ( "k8s.io/minikube/pkg/util/retry" ) -// remote artifacts that must exist for minikube to function properly. The sign of a previously working installation. -// NOTE: /etc is not persistent across restarts, so don't bother checking there -var expectedArtifacts = []string{ - "/var/lib/kubelet/kubeadm-flags.env", - "/var/lib/kubelet/config.yaml", - bsutil.EtcdDataDir(), -} - -// SkipAdditionalPreflights are additional preflights we skip depending on the runtime in use. -var SkipAdditionalPreflights = map[string][]string{} - // Bootstrapper is a bootstrapper using kubeadm type Bootstrapper struct { c command.Runner + k8sClient *kubernetes.Clientset // kubernetes client used to verify pods inside cluster contextName string } @@ -82,7 +70,7 @@ func NewBootstrapper(api libmachine.API) (*Bootstrapper, error) { if err != nil { return nil, errors.Wrap(err, "command runner") } - return &Bootstrapper{c: runner, contextName: name}, nil + return &Bootstrapper{c: runner, contextName: name, k8sClient: nil}, nil } // GetKubeletStatus returns the kubelet status @@ -105,23 +93,7 @@ func (k *Bootstrapper) GetKubeletStatus() (string, error) { // GetAPIServerStatus returns the api-server status func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string, error) { - url := fmt.Sprintf("https://%s:%d/healthz", ip, apiserverPort) - // To avoid: x509: certificate signed by unknown authority - tr := &http.Transport{ - Proxy: nil, // To avoid connectiv issue if http(s)_proxy is set. - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client := &http.Client{Transport: tr} - resp, err := client.Get(url) - // Connection refused, usually. - if err != nil { - return state.Stopped.String(), nil - } - if resp.StatusCode != http.StatusOK { - glog.Warningf("%s response: %v %+v", url, err, resp) - return state.Error.String(), nil - } - return state.Running.String(), nil + return kverify.APIServerStatus(ip, apiserverPort) } // LogCommands returns a map of log type to a command which will display that log. @@ -166,16 +138,10 @@ func (k *Bootstrapper) createCompatSymlinks() error { return nil } -func (k *Bootstrapper) existingConfig() error { - args := append([]string{"ls"}, expectedArtifacts...) - _, err := k.c.RunCmd(exec.Command("sudo", args...)) - return err -} - // StartCluster starts the cluster func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { - err := k.existingConfig() - if err == nil { + err := bsutil.ExistingConfig(k.c) + if err == nil { // if there is an existing cluster don't reconfigure it return k.restartCluster(k8s) } glog.Infof("existence check: %v", err) @@ -208,7 +174,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { "Port-10250", // For "none" users who already have a kubelet online "Swap", // For "none" users who have swap configured } - ignore = append(ignore, SkipAdditionalPreflights[r.Name()]...) + ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...) // Allow older kubeadm versions to function with newer Docker releases. if version.LT(semver.MustParse("1.13.0")) { @@ -222,191 +188,65 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { } glog.Infof("Configuring cluster permissions ...") - elevate := func() error { client, err := k.client(k8s) if err != nil { return err } - return elevateKubeSystemPrivileges(client) + return bsutil.ElevateKubeSystemPrivileges(client) } if err := retry.Expo(elevate, time.Millisecond*500, 120*time.Second); err != nil { return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges") } - if err := k.adjustResourceLimits(); err != nil { + if err := bsutil.AdjustResourceLimits(k.c); err != nil { glog.Warningf("unable to adjust resource limits: %v", err) } return nil } -// adjustResourceLimits makes fine adjustments to pod resources that aren't possible via kubeadm config. -func (k *Bootstrapper) adjustResourceLimits() error { - rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", "cat /proc/$(pgrep kube-apiserver)/oom_adj")) - if err != nil { - return errors.Wrapf(err, "oom_adj check cmd %s. ", rr.Command()) - } - glog.Infof("apiserver oom_adj: %s", rr.Stdout.String()) - // oom_adj is already a negative number - if strings.HasPrefix(rr.Stdout.String(), "-") { - return nil - } - glog.Infof("adjusting apiserver oom_adj to -10") - - // Prevent the apiserver from OOM'ing before other pods, as it is our gateway into the cluster. - // It'd be preferable to do this via Kubernetes, but kubeadm doesn't have a way to set pod QoS. - if _, err = k.c.RunCmd(exec.Command("/bin/bash", "-c", "echo -10 | sudo tee /proc/$(pgrep kube-apiserver)/oom_adj")); err != nil { - return errors.Wrap(err, fmt.Sprintf("oom_adj adjust")) - } - - return nil -} - -func addAddons(files *[]assets.CopyableFile, data interface{}) error { - // add addons to file list - // custom addons - if err := assets.AddMinikubeDirAssets(files); err != nil { - return errors.Wrap(err, "adding minikube dir assets") - } - // bundled addons - for _, addonBundle := range assets.Addons { - if isEnabled, err := addonBundle.IsEnabled(); err == nil && isEnabled { - for _, addon := range addonBundle.Assets { - if addon.IsTemplate() { - addonFile, err := addon.Evaluate(data) - if err != nil { - return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName()) - } - - *files = append(*files, addonFile) - } else { - *files = append(*files, addon) - } - } - } else if err != nil { - return nil - } +// client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver +func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) { + if k.k8sClient != nil { + return k.k8sClient, nil } - return nil -} - -// client returns a Kubernetes client to use to speak to a kubeadm launched apiserver -func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) { config, err := kapi.ClientConfig(k.contextName) if err != nil { return nil, errors.Wrap(err, "client config") } - endpoint := fmt.Sprintf("https://%s:%d", k8s.NodeIP, k8s.NodePort) + endpoint := fmt.Sprintf("https://%s", net.JoinHostPort(k8s.NodeIP, strconv.Itoa(k8s.NodePort))) if config.Host != endpoint { glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint) config.Host = endpoint } - - return kubernetes.NewForConfig(config) -} - -func (k *Bootstrapper) waitForAPIServerProcess(start time.Time, timeout time.Duration) error { - glog.Infof("waiting for apiserver process to appear ...") - err := wait.PollImmediate(time.Second*1, timeout, func() (bool, error) { - if time.Since(start) > timeout { - return false, fmt.Errorf("cluster wait timed out during process check") - } - rr, ierr := k.c.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver")) - if ierr != nil { - glog.Warningf("pgrep apiserver: %v cmd: %s", ierr, rr.Command()) - return false, nil - } - return true, nil - }) - if err != nil { - return fmt.Errorf("apiserver process never appeared") - } - glog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start)) - return nil -} - -func (k *Bootstrapper) waitForAPIServerHealthz(start time.Time, k8s config.KubernetesConfig, timeout time.Duration) error { - glog.Infof("waiting for apiserver healthz status ...") - hStart := time.Now() - healthz := func() (bool, error) { - if time.Since(start) > timeout { - return false, fmt.Errorf("cluster wait timed out during healthz check") - } - - status, err := k.GetAPIServerStatus(net.ParseIP(k8s.NodeIP), k8s.NodePort) - if err != nil { - glog.Warningf("status: %v", err) - return false, nil - } - if status != "Running" { - return false, nil - } - return true, nil - } - - if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, healthz); err != nil { - return fmt.Errorf("apiserver healthz never reported healthy") - } - glog.Infof("duration metric: took %s to wait for apiserver healthz status ...", time.Since(hStart)) - return nil -} - -func (k *Bootstrapper) waitForSystemPods(start time.Time, k8s config.KubernetesConfig, timeout time.Duration) error { - glog.Infof("waiting for kube-system pods to appear ...") - pStart := time.Now() - client, err := k.client(k8s) - if err != nil { - return errors.Wrap(err, "client") - } - - podStart := time.Time{} - podList := func() (bool, error) { - if time.Since(start) > timeout { - return false, fmt.Errorf("cluster wait timed out during pod check") - } - // Wait for any system pod, as waiting for apiserver may block until etcd - pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{}) - if len(pods.Items) < 2 { - podStart = time.Time{} - return false, nil - } - if err != nil { - podStart = time.Time{} - return false, nil - } - if podStart.IsZero() { - podStart = time.Now() - } - - glog.Infof("%d kube-system pods found since %s", len(pods.Items), podStart) - if time.Since(podStart) > 2*kconst.APICallRetryInterval { - glog.Infof("stability requirement met, returning") - return true, nil - } - return false, nil - } - if err = wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, podList); err != nil { - return fmt.Errorf("apiserver never returned a pod list") + c, err := kubernetes.NewForConfig(config) + if err == nil { + k.k8sClient = c } - glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart)) - return nil + return c, err } // WaitForCluster blocks until the cluster appears to be healthy func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time.Duration) error { start := time.Now() out.T(out.Waiting, "Waiting for cluster to come online ...") - if err := k.waitForAPIServerProcess(start, timeout); err != nil { + if err := kverify.APIServerProcess(k.c, start, timeout); err != nil { return err } - if err := k.waitForAPIServerHealthz(start, k8s, timeout); err != nil { + if err := kverify.APIServerIsRunning(start, k8s.NodeIP, k8s.NodePort, timeout); err != nil { return err } - return k.waitForSystemPods(start, k8s, timeout) + + c, err := k.client(k8s) + if err != nil { + return errors.Wrap(err, "get k8s client") + } + + return kverify.SystemPods(c, start, k8s.NodeIP, k8s.NodePort, timeout) } // restartCluster restarts the Kubernetes cluster configured by kubeadm @@ -451,10 +291,16 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { } // We must ensure that the apiserver is healthy before proceeding - if err := k.waitForAPIServerHealthz(time.Now(), k8s, kconst.DefaultControlPlaneTimeout); err != nil { + if err := kverify.APIServerProcess(k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil { return errors.Wrap(err, "apiserver healthz") } - if err := k.waitForSystemPods(time.Now(), k8s, kconst.DefaultControlPlaneTimeout); err != nil { + + client, err := k.client(k8s) + if err != nil { + return errors.Wrap(err, "getting k8s client") + } + + if err := kverify.SystemPods(client, time.Now(), k8s.NodeIP, k8s.NodePort, kconst.DefaultControlPlaneTimeout); err != nil { return errors.Wrap(err, "system pods") } @@ -463,7 +309,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { return errors.Wrapf(err, fmt.Sprintf("addon phase cmd:%q", rr.Command())) } - if err := k.adjustResourceLimits(); err != nil { + if err := bsutil.AdjustResourceLimits(k.c); err != nil { glog.Warningf("unable to adjust resource limits: %v", err) } return nil @@ -559,7 +405,7 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { } files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile) - if err := addAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil { + if err := bsutil.AddAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil { return errors.Wrap(err, "adding addons") } for _, f := range files { diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go index e116e2d50353..97492a33cf53 100644 --- a/pkg/minikube/cluster/cluster.go +++ b/pkg/minikube/cluster/cluster.go @@ -24,6 +24,7 @@ import ( "math" "net" "os/exec" + "path/filepath" "regexp" "strconv" "strings" @@ -38,6 +39,7 @@ import ( "github.com/docker/machine/libmachine/ssh" "github.com/docker/machine/libmachine/state" "github.com/golang/glog" + "github.com/juju/mutex" "github.com/pkg/errors" "github.com/shirou/gopsutil/cpu" "github.com/shirou/gopsutil/disk" @@ -52,7 +54,7 @@ import ( "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/registry" - pkgutil "k8s.io/minikube/pkg/util" + "k8s.io/minikube/pkg/util/lock" "k8s.io/minikube/pkg/util/retry" ) @@ -89,6 +91,17 @@ func CacheISO(config cfg.MachineConfig) error { // StartHost starts a host VM. func StartHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error) { + // Prevent machine-driver boot races, as well as our own certificate race + releaser, err := acquireMachinesLock(config.Name) + if err != nil { + return nil, errors.Wrap(err, "boot lock") + } + start := time.Now() + defer func() { + glog.Infof("releasing machines lock for %q, held for %s", config.Name, time.Since(start)) + releaser.Release() + }() + exists, err := api.Exists(config.Name) if err != nil { return nil, errors.Wrapf(err, "exists: %s", config.Name) @@ -139,6 +152,21 @@ func StartHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error) return h, nil } +// acquireMachinesLock protects against code that is not parallel-safe (libmachine, cert setup) +func acquireMachinesLock(name string) (mutex.Releaser, error) { + spec := lock.PathMutexSpec(filepath.Join(localpath.MiniPath(), "machines")) + // NOTE: Provisioning generally completes within 60 seconds + spec.Timeout = 10 * time.Minute + + glog.Infof("acquiring machines lock for %s: %+v", name, spec) + start := time.Now() + r, err := mutex.Acquire(spec) + if err == nil { + glog.Infof("acquired machines lock for %q in %s", name, time.Since(start)) + } + return r, err +} + // configureHost handles any post-powerup configuration required func configureHost(h *host.Host, e *engine.Options) error { start := time.Now() @@ -343,7 +371,7 @@ func GetHostDriverIP(api libmachine.API, machineName string) (net.IP, error) { func engineOptions(config cfg.MachineConfig) *engine.Options { o := engine.Options{ Env: config.DockerEnv, - InsecureRegistry: append([]string{pkgutil.DefaultServiceCIDR}, config.InsecureRegistry...), + InsecureRegistry: append([]string{constants.DefaultServiceCIDR}, config.InsecureRegistry...), RegistryMirror: config.RegistryMirror, ArbitraryFlags: config.DockerOpt, InstallURL: drivers.DefaultEngineInstallURL, @@ -419,13 +447,8 @@ func showRemoteOsRelease(driver drivers.Driver) { glog.Infof("Provisioned with %s", osReleaseInfo.PrettyName) } -func createHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error) { - if config.VMDriver == driver.VMwareFusion && viper.GetBool(cfg.ShowDriverDeprecationNotification) { - out.WarningT(`The vmwarefusion driver is deprecated and support for it will be removed in a future release. - Please consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver. - See https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information. - To disable this message, run [minikube config set ShowDriverDeprecationNotification false]`) - } +// showHostInfo shows host information +func showHostInfo(config cfg.MachineConfig) { if driver.BareMetal(config.VMDriver) { info, err := getHostInfo() if err == nil { @@ -439,6 +462,16 @@ func createHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error } else { out.T(out.StartingVM, "Creating {{.driver_name}} VM (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"driver_name": config.VMDriver, "number_of_cpus": config.CPUs, "memory_size": config.Memory, "disk_size": config.DiskSize}) } +} + +func createHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error) { + if config.VMDriver == driver.VMwareFusion && viper.GetBool(cfg.ShowDriverDeprecationNotification) { + out.WarningT(`The vmwarefusion driver is deprecated and support for it will be removed in a future release. + Please consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver. + See https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information. + To disable this message, run [minikube config set ShowDriverDeprecationNotification false]`) + } + showHostInfo(config) def := registry.Driver(config.VMDriver) if def.Empty() { return nil, fmt.Errorf("unsupported/missing driver: %s", config.VMDriver) diff --git a/pkg/minikube/command/kic_runner.go b/pkg/minikube/command/kic_runner.go index 1c7a3b665037..8dbcf11e1317 100644 --- a/pkg/minikube/command/kic_runner.go +++ b/pkg/minikube/command/kic_runner.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "os" "os/exec" + "path" "time" "github.com/golang/glog" @@ -159,9 +160,9 @@ func (k *kicRunner) Copy(f assets.CopyableFile) error { if out, err := exec.Command(k.ociBin, "cp", assetName, destination).CombinedOutput(); err != nil { return errors.Wrapf(err, "error copying %s into node, output: %s", f.GetAssetName(), string(out)) } - - if _, err := k.RunCmd(exec.Command("chmod", f.GetPermissions(), f.GetTargetDir())); err != nil { - return errors.Wrap(err, "failed to chmod file permissions") + fp := path.Join(f.GetTargetDir(), f.GetTargetName()) + if _, err := k.RunCmd(exec.Command("sudo", "chmod", f.GetPermissions(), fp)); err != nil { + return errors.Wrapf(err, "failed to chmod file permissions %s", fp) } return nil } diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go index db4efc1f2cf0..7abd2a8d8119 100644 --- a/pkg/minikube/config/config.go +++ b/pkg/minikube/config/config.go @@ -127,9 +127,15 @@ func Load(profile string) (*MachineConfig, error) { return DefaultLoader.LoadConfigFromFile(profile) } +// Write writes the kubernetes and machine config for the current machine +func Write(profile string, cc *MachineConfig) error { + return DefaultLoader.WriteConfigToFile(profile, cc) +} + // Loader loads the kubernetes and machine config based on the machine profile name type Loader interface { LoadConfigFromFile(profile string, miniHome ...string) (*MachineConfig, error) + WriteConfigToFile(profileName string, cc *MachineConfig, miniHome ...string) error } type simpleConfigLoader struct{} @@ -156,3 +162,13 @@ func (c *simpleConfigLoader) LoadConfigFromFile(profileName string, miniHome ... } return &cc, nil } + +func (c *simpleConfigLoader) WriteConfigToFile(profileName string, cc *MachineConfig, miniHome ...string) error { + // Move to profile package + path := profileFilePath(profileName, miniHome...) + contents, err := json.MarshalIndent(cc, "", " ") + if err != nil { + return err + } + return ioutil.WriteFile(path, contents, 0644) +} diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 11228b38ff83..18bd9a75b4f3 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -64,7 +64,8 @@ type MachineConfig struct { KubernetesConfig KubernetesConfig HostOnlyNicType string // Only used by virtualbox NatNicType string // Only used by virtualbox - NodeBindPort int32 // Only used by kic + Addons map[string]bool + NodeBindPort int32 // Only used by kic } // KubernetesConfig contains the parameters used to configure the VM Kubernetes. @@ -80,8 +81,8 @@ type KubernetesConfig struct { ContainerRuntime string CRISocket string NetworkPlugin string - FeatureGates string - ServiceCIDR string + FeatureGates string // https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ + ServiceCIDR string // the subnet which kubernetes services will be deployed to ImageRepository string ExtraOptions ExtraOptionSlice diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 9d328de7f3d2..85f6d40a7789 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -33,6 +33,8 @@ const ( APIServerName = "minikubeCA" // ClusterDNSDomain is the default DNS domain ClusterDNSDomain = "cluster.local" + // DefaultServiceCIDR is The CIDR to be used for service cluster IPs + DefaultServiceCIDR = "10.96.0.0/12" ) // DefaultMinipath is the default Minikube path (under the home directory) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index d431ee2636ee..3e91a45edc55 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -267,7 +267,7 @@ func (r *Containerd) StopContainers(ids []string) error { // ContainerLogCmd returns the command to retrieve the log for a container based on ID func (r *Containerd) ContainerLogCmd(id string, len int, follow bool) string { - return criContainerLogCmd(id, len, follow) + return criContainerLogCmd(r.Runner, id, len, follow) } // SystemLogCmd returns the command to retrieve system logs diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index 4cb25646d5a3..ab6aa8014e74 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -29,16 +29,27 @@ import ( "k8s.io/minikube/pkg/minikube/command" ) +// getCrictlPath returns the absolute path of crictl +func getCrictlPath(cr CommandRunner) string { + cmd := "crictl" + rr, err := cr.RunCmd(exec.Command("which", cmd)) + if err != nil { + return cmd + } + return strings.Split(rr.Stdout.String(), "\n")[0] +} + // listCRIContainers returns a list of containers using crictl func listCRIContainers(cr CommandRunner, filter string) ([]string, error) { var err error var rr *command.RunResult state := "Running" + crictl := getCrictlPath(cr) if filter != "" { - c := exec.Command("sudo", "crictl", "ps", "-a", fmt.Sprintf("--name=%s", filter), fmt.Sprintf("--state=%s", state), "--quiet") + c := exec.Command("sudo", crictl, "ps", "-a", fmt.Sprintf("--name=%s", filter), fmt.Sprintf("--state=%s", state), "--quiet") rr, err = cr.RunCmd(c) } else { - rr, err = cr.RunCmd(exec.Command("sudo", "crictl", "ps", "-a", fmt.Sprintf("--state=%s", state), "--quiet")) + rr, err = cr.RunCmd(exec.Command("sudo", crictl, "ps", "-a", fmt.Sprintf("--state=%s", state), "--quiet")) } if err != nil { return nil, err @@ -59,7 +70,8 @@ func killCRIContainers(cr CommandRunner, ids []string) error { } glog.Infof("Killing containers: %s", ids) - args := append([]string{"crictl", "rm"}, ids...) + crictl := getCrictlPath(cr) + args := append([]string{crictl, "rm"}, ids...) c := exec.Command("sudo", args...) if _, err := cr.RunCmd(c); err != nil { return errors.Wrap(err, "kill cri containers.") @@ -73,7 +85,9 @@ func stopCRIContainers(cr CommandRunner, ids []string) error { return nil } glog.Infof("Stopping containers: %s", ids) - args := append([]string{"crictl", "rm"}, ids...) + + crictl := getCrictlPath(cr) + args := append([]string{crictl, "rm"}, ids...) c := exec.Command("sudo", args...) if _, err := cr.RunCmd(c); err != nil { return errors.Wrap(err, "stop cri containers") @@ -105,9 +119,12 @@ image-endpoint: unix://{{.Socket}} } // criContainerLogCmd returns the command to retrieve the log for a container based on ID -func criContainerLogCmd(id string, len int, follow bool) string { +func criContainerLogCmd(cr CommandRunner, id string, len int, follow bool) string { + crictl := getCrictlPath(cr) var cmd strings.Builder - cmd.WriteString("sudo crictl logs ") + cmd.WriteString("sudo ") + cmd.WriteString(crictl) + cmd.WriteString(" logs ") if len > 0 { cmd.WriteString(fmt.Sprintf("--tail %d ", len)) } diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 13ac34cf00bf..f95d318ae27f 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -188,7 +188,7 @@ func (r *CRIO) StopContainers(ids []string) error { // ContainerLogCmd returns the command to retrieve the log for a container based on ID func (r *CRIO) ContainerLogCmd(id string, len int, follow bool) string { - return criContainerLogCmd(id, len, follow) + return criContainerLogCmd(r.Runner, id, len, follow) } // SystemLogCmd returns the command to retrieve system logs diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index d59499d40e08..4c0a01186656 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -102,6 +102,12 @@ func New(c Config) (Manager, error) { } } +// ContainerStatusCommand works across container runtimes with good formatting +func ContainerStatusCommand() string { + // Fallback to 'docker ps' if it fails (none driver) + return "sudo `which crictl || echo crictl` ps -a || sudo docker ps -a" +} + // disableOthers disables all other runtimes except for me. func disableOthers(me Manager, cr CommandRunner) error { // valid values returned by manager.Name() diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go index 13ed8bdde664..971ed22d2c9c 100644 --- a/pkg/minikube/cruntime/cruntime_test.go +++ b/pkg/minikube/cruntime/cruntime_test.go @@ -143,9 +143,11 @@ func (f *FakeRunner) RunCmd(cmd *exec.Cmd) (*command.RunResult, error) { switch bin { case "systemctl": return buffer(f.systemctl(args, root)) + case "which": + return buffer(f.which(args, root)) case "docker": return buffer(f.docker(args, root)) - case "crictl": + case "crictl", "/usr/bin/crictl": return buffer(f.crictl(args, root)) case "crio": return buffer(f.crio(args, root)) @@ -321,6 +323,13 @@ func (f *FakeRunner) systemctl(args []string, root bool) (string, error) { // no return out, nil } +// which is a fake implementation of which +func (f *FakeRunner) which(args []string, root bool) (string, error) { // nolint result 0 (string) is always "" + command := args[0] + path := fmt.Sprintf("/usr/bin/%s", command) + return path, nil +} + func TestVersion(t *testing.T) { var tests = []struct { runtime string diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 1aea66e46621..c084b4984639 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -22,6 +22,7 @@ import ( "sort" "github.com/golang/glog" + "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/registry" ) @@ -81,7 +82,7 @@ func BareMetal(name string) bool { // FlagHints are hints for what default options should be used for this driver type FlagHints struct { - ExtraOptions string + ExtraOptions []string CacheImages bool ContainerRuntime string Bootstrapper string @@ -89,24 +90,26 @@ type FlagHints struct { // FlagDefaults returns suggested defaults based on a driver func FlagDefaults(name string) FlagHints { + fh := FlagHints{} if name != None { - fh := FlagHints{CacheImages: true} + fh.CacheImages = true // only for kic, till other run-times are available we auto-set containerd. if name == Docker { fh.ContainerRuntime = "containerd" fh.Bootstrapper = bootstrapper.KIC + fh.ExtraOptions = append(fh.ExtraOptions, fmt.Sprintf("kubeadm.pod-network-cidr=%s", kic.DefaultPodCIDR)) } return fh } - extraOpts := "" + fh.CacheImages = false + // if specifc linux add this option for systemd work on none driver if _, err := os.Stat(systemdResolvConf); err == nil { - extraOpts = fmt.Sprintf("kubelet.resolv-conf=%s", systemdResolvConf) - } - return FlagHints{ - ExtraOptions: extraOpts, - CacheImages: false, + noneEO := fmt.Sprintf("kubelet.resolv-conf=%s", systemdResolvConf) + fh.ExtraOptions = append(fh.ExtraOptions, noneEO) + return fh } + return fh } // Choices returns a list of drivers which are possible on this system @@ -168,3 +171,10 @@ func Choose(requested string, options []registry.DriverState) (registry.DriverSt func Status(name string) registry.State { return registry.Status(name) } + +// SetLibvirtURI sets the URI to perform libvirt health checks against +func SetLibvirtURI(v string) { + glog.Infof("Setting default libvirt URI to %s", v) + os.Setenv("LIBVIRT_DEFAULT_URI", v) + +} diff --git a/pkg/minikube/driver/driver_test.go b/pkg/minikube/driver/driver_test.go index 7f07f9d62e8f..5aad1274b42d 100644 --- a/pkg/minikube/driver/driver_test.go +++ b/pkg/minikube/driver/driver_test.go @@ -72,7 +72,7 @@ func TestFlagDefaults(t *testing.T) { expected = FlagHints{ CacheImages: false, - ExtraOptions: fmt.Sprintf("kubelet.resolv-conf=%s", tf.Name()), + ExtraOptions: []string{fmt.Sprintf("kubelet.resolv-conf=%s", tf.Name())}, } systemdResolvConf = tf.Name() if diff := cmp.Diff(FlagDefaults(None), expected); diff != "" { diff --git a/pkg/minikube/driver/install.go b/pkg/minikube/driver/install.go index 1d0b856e3987..2732fe6f36f0 100644 --- a/pkg/minikube/driver/install.go +++ b/pkg/minikube/driver/install.go @@ -23,14 +23,17 @@ import ( "path/filepath" "regexp" "strings" + "time" "github.com/blang/semver" "github.com/golang/glog" "github.com/hashicorp/go-getter" + "github.com/juju/mutex" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/util" + "k8s.io/minikube/pkg/util/lock" ) // InstallOrUpdate downloads driver if it is not present, or updates it if there's a newer version @@ -40,6 +43,17 @@ func InstallOrUpdate(name string, directory string, v semver.Version, interactiv } executable := fmt.Sprintf("docker-machine-driver-%s", name) + + // Lock before we check for existence to avoid thundering herd issues + spec := lock.PathMutexSpec(executable) + spec.Timeout = 10 * time.Minute + glog.Infof("acquiring lock: %+v", spec) + releaser, err := mutex.Acquire(spec) + if err != nil { + return errors.Wrapf(err, "unable to acquire lock for %+v", spec) + } + defer releaser.Release() + exists := driverExists(executable) path, err := validateDriver(executable, v) if !exists || (err != nil && autoUpdate) { diff --git a/pkg/minikube/exit/exit.go b/pkg/minikube/exit/exit.go index c3650b6cb835..4ed989d2a692 100644 --- a/pkg/minikube/exit/exit.go +++ b/pkg/minikube/exit/exit.go @@ -72,7 +72,7 @@ func WithProblem(msg string, p *problem.Problem) { out.ErrT(out.Empty, "") out.FatalT(msg) p.Display() - if !p.ShowIssueLink { + if p.ShowIssueLink { out.ErrT(out.Empty, "") out.ErrT(out.Sad, "If the above advice does not help, please let us know: ") out.ErrT(out.URL, "https://github.com/kubernetes/minikube/issues/new/choose") diff --git a/pkg/minikube/image/cache.go b/pkg/minikube/image/cache.go index 93ac59e59e01..3a29e779b5f1 100644 --- a/pkg/minikube/image/cache.go +++ b/pkg/minikube/image/cache.go @@ -25,10 +25,12 @@ import ( "github.com/golang/glog" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/juju/mutex" "github.com/pkg/errors" "golang.org/x/sync/errgroup" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/util/lock" ) // DeleteFromCacheDir deletes tar files stored in cache dir @@ -72,23 +74,33 @@ func SaveToDir(images []string, cacheDir string) error { } // saveToTarFile caches an image -func saveToTarFile(image, dst string) error { +func saveToTarFile(image, rawDest string) error { start := time.Now() defer func() { - glog.Infof("cache image %q -> %s to local destination -> %q", image, dst, time.Since(start)) + glog.Infof("cache image %q -> %q took %s", image, rawDest, time.Since(start)) }() - if _, err := os.Stat(dst); err == nil { - glog.Infof("%s exists", dst) - return nil + // OS-specific mangling of destination path + dst, err := localpath.DstPath(rawDest) + if err != nil { + return errors.Wrap(err, "getting destination path") } - dstPath, err := localpath.DstPath(dst) + spec := lock.PathMutexSpec(dst) + spec.Timeout = 10 * time.Minute + glog.Infof("acquiring lock: %+v", spec) + releaser, err := mutex.Acquire(spec) if err != nil { - return errors.Wrap(err, "getting destination path") + return errors.Wrapf(err, "unable to acquire lock for %+v", spec) + } + defer releaser.Release() + + if _, err := os.Stat(dst); err == nil { + glog.Infof("%s exists", dst) + return nil } - if err := os.MkdirAll(filepath.Dir(dstPath), 0777); err != nil { + if err := os.MkdirAll(filepath.Dir(dst), 0777); err != nil { return errors.Wrapf(err, "making cache image directory: %s", dst) } @@ -101,8 +113,8 @@ func saveToTarFile(image, dst string) error { if err != nil { glog.Warningf("unable to retrieve image: %v", err) } - glog.Infoln("opening: ", dstPath) - f, err := ioutil.TempFile(filepath.Dir(dstPath), filepath.Base(dstPath)+".*.tmp") + glog.Infoln("opening: ", dst) + f, err := ioutil.TempFile(filepath.Dir(dst), filepath.Base(dst)+".*.tmp") if err != nil { return err } @@ -128,7 +140,7 @@ func saveToTarFile(image, dst string) error { if err != nil { return errors.Wrap(err, "close") } - err = os.Rename(f.Name(), dstPath) + err = os.Rename(f.Name(), dst) if err != nil { return errors.Wrap(err, "rename") } diff --git a/pkg/minikube/localpath/localpath.go b/pkg/minikube/localpath/localpath.go index b5a00092c833..eb61a0d2f17e 100644 --- a/pkg/minikube/localpath/localpath.go +++ b/pkg/minikube/localpath/localpath.go @@ -61,8 +61,7 @@ func MachinePath(machine string, miniHome ...string) string { return filepath.Join(miniPath, "machines", machine) } -// SanitizeCacheDir -// # ParseReference cannot have a : in the directory path +// SanitizeCacheDir returns a path without special characters func SanitizeCacheDir(image string) string { if runtime.GOOS == "windows" && hasWindowsDriveLetter(image) { // not sanitize Windows drive letter. @@ -70,6 +69,7 @@ func SanitizeCacheDir(image string) string { glog.Infof("windows sanitize: %s -> %s", image, s) return s } + // ParseReference cannot have a : in the directory path return strings.Replace(image, ":", "_", -1) } diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go index a62a43ace25e..f44455df2b00 100644 --- a/pkg/minikube/logs/logs.go +++ b/pkg/minikube/logs/logs.go @@ -186,8 +186,6 @@ func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, length int, f } } cmds[r.Name()] = r.SystemLogCmd(length) - // Works across container runtimes with good formatting - // Fallback to 'docker ps' if it fails (none driver) - cmds["container status"] = "sudo crictl ps -a || sudo docker ps -a" + cmds["container status"] = cruntime.ContainerStatusCommand() return cmds } diff --git a/pkg/minikube/out/style.go b/pkg/minikube/out/style.go index b14c9d5883e2..3695ed416725 100644 --- a/pkg/minikube/out/style.go +++ b/pkg/minikube/out/style.go @@ -113,6 +113,7 @@ var styles = map[StyleEnum]style{ Unmount: {Prefix: "🔥 "}, MountOptions: {Prefix: "💾 "}, Fileserver: {Prefix: "🚀 ", OmitNewline: true}, + DryRun: {Prefix: "🏜️ "}, } // Add a prefix to a string diff --git a/pkg/minikube/out/style_enum.go b/pkg/minikube/out/style_enum.go index 0bc8ac182279..81e232b14194 100644 --- a/pkg/minikube/out/style_enum.go +++ b/pkg/minikube/out/style_enum.go @@ -84,4 +84,5 @@ const ( Empty Workaround Sparkle + DryRun ) diff --git a/pkg/minikube/proxy/proxy_test.go b/pkg/minikube/proxy/proxy_test.go index 4e76023f9162..4475905c508c 100644 --- a/pkg/minikube/proxy/proxy_test.go +++ b/pkg/minikube/proxy/proxy_test.go @@ -33,6 +33,7 @@ func TestIsValidEnv(t *testing.T) { {"", false}, {"HTTPS-PROXY", false}, {"NOPROXY", false}, + {"http_proxy", true}, } for _, tc := range testCases { t.Run(tc.env, func(t *testing.T) { diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index 9abfc1d54627..42fc292b8711 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -21,8 +21,8 @@ import ( "os/exec" "github.com/docker/machine/libmachine/drivers" - "github.com/golang/glog" "k8s.io/minikube/pkg/drivers/kic" + "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" @@ -33,7 +33,7 @@ func init() { if err := registry.Register(registry.DriverDef{ Name: driver.Docker, Config: configure, - Init: func() drivers.Driver { return kic.NewDriver(kic.Config{}) }, + Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIBinary: oci.Docker}) }, Status: status, Priority: registry.Discouraged, // experimental }); err != nil { @@ -42,18 +42,14 @@ func init() { } func configure(mc config.MachineConfig) interface{} { - img, err := kic.ImageForVersion(mc.KubernetesConfig.KubernetesVersion) - if err != nil { - glog.Errorf("err to getting kic image for %s: imgesha:%s", img, mc.KubernetesConfig.KubernetesVersion) - } return kic.NewDriver(kic.Config{ - MachineName: mc.Name, - StorePath: localpath.MiniPath(), - ImageDigest: img, - CPU: mc.CPUs, - Memory: mc.Memory, - APIServerPort: mc.NodeBindPort, - OCIBinary: "docker", + MachineName: mc.Name, + StorePath: localpath.MiniPath(), + ImageDigest: kic.BaseImage, + CPU: mc.CPUs, + Memory: mc.Memory, + HostBindPort: mc.KubernetesConfig.NodePort, + OCIBinary: oci.Docker, }) } diff --git a/pkg/minikube/registry/drvs/kvm2/kvm2.go b/pkg/minikube/registry/drvs/kvm2/kvm2.go index 86bf44c96844..89c5e7781be9 100644 --- a/pkg/minikube/registry/drvs/kvm2/kvm2.go +++ b/pkg/minikube/registry/drvs/kvm2/kvm2.go @@ -20,6 +20,7 @@ package kvm2 import ( "fmt" + "os" "os/exec" "path/filepath" "strings" @@ -86,13 +87,25 @@ func configure(mc config.MachineConfig) interface{} { } } +// defaultURI returns the QEMU URI to connect to for health checks +func defaultURI() string { + u := os.Getenv("LIBVIRT_DEFAULT_URI") + if u != "" { + return u + } + return "qemu:///system" +} + func status() registry.State { path, err := exec.LookPath("virsh") if err != nil { return registry.State{Error: err, Fix: "Install libvirt", Doc: docURL} } + // On Ubuntu 19.10 (libvirt 5.4), this fails if LIBVIRT_DEFAULT_URI is unset cmd := exec.Command(path, "domcapabilities", "--virttype", "kvm") + cmd.Env = append(os.Environ(), fmt.Sprintf("LIBVIRT_DEFAULT_URI=%s", defaultURI())) + out, err := cmd.CombinedOutput() if err != nil { return registry.State{ @@ -104,6 +117,7 @@ func status() registry.State { } cmd = exec.Command("virsh", "list") + cmd.Env = append(os.Environ(), fmt.Sprintf("LIBVIRT_DEFAULT_URI=%s", defaultURI())) out, err = cmd.CombinedOutput() if err != nil { return registry.State{ diff --git a/pkg/minikube/service/service_test.go b/pkg/minikube/service/service_test.go index fe46a3df6329..47fd341f42e9 100644 --- a/pkg/minikube/service/service_test.go +++ b/pkg/minikube/service/service_test.go @@ -44,6 +44,7 @@ import ( "k8s.io/minikube/pkg/minikube/tests" ) +// Mock Kubernetes client getter - NOT THREAD SAFE type MockClientGetter struct { servicesMap map[string]typed_core.ServiceInterface endpointsMap map[string]typed_core.EndpointsInterface @@ -76,6 +77,7 @@ func (m *MockCoreClient) Services(namespace string) typed_core.ServiceInterface return m.servicesMap[namespace] } +// Mock Kubernetes client - NOT THREAD SAFE type MockCoreClient struct { fake.FakeCoreV1 servicesMap map[string]typed_core.ServiceInterface @@ -466,8 +468,6 @@ func TestGetServiceURLs(t *testing.T) { for _, test := range tests { test := test t.Run(test.description, func(t *testing.T) { - t.Parallel() - K8s = &MockClientGetter{ servicesMap: serviceNamespaces, endpointsMap: endpointNamespaces, @@ -536,7 +536,6 @@ func TestGetServiceURLsForService(t *testing.T) { defer revertK8sClient(K8s) for _, test := range tests { t.Run(test.description, func(t *testing.T) { - t.Parallel() K8s = &MockClientGetter{ servicesMap: serviceNamespaces, endpointsMap: endpointNamespaces, diff --git a/pkg/minikube/tests/ssh_mock.go b/pkg/minikube/tests/ssh_mock.go index ebb9206fc194..2f08d269b463 100644 --- a/pkg/minikube/tests/ssh_mock.go +++ b/pkg/minikube/tests/ssh_mock.go @@ -122,11 +122,14 @@ func (s *SSHServer) handleIncomingConnection(c net.Conn) { func (s *SSHServer) handleRequest(channel ssh.Channel, req *ssh.Request) { go func() { - if _, err := io.Copy(s.Transfers, channel); err != nil { + // Explicitly copy buffer contents to avoid data race + b := s.Transfers.Bytes() + if _, err := io.Copy(bytes.NewBuffer(b), channel); err != nil { s.t.Errorf("copy failed: %v", err) } channel.Close() }() + switch req.Type { case "exec": s.t.Logf("exec request received: %+v", req) diff --git a/pkg/minikube/tunnel/test_doubles.go b/pkg/minikube/tunnel/test_doubles.go index 414910a9301b..5ac487459366 100644 --- a/pkg/minikube/tunnel/test_doubles.go +++ b/pkg/minikube/tunnel/test_doubles.go @@ -86,6 +86,10 @@ type stubConfigLoader struct { e error } +func (l *stubConfigLoader) WriteConfigToFile(profileName string, cc *config.MachineConfig, miniHome ...string) error { + return l.e +} + func (l *stubConfigLoader) LoadConfigFromFile(profile string, miniHome ...string) (*config.MachineConfig, error) { return l.c, l.e } diff --git a/pkg/util/constants.go b/pkg/util/constants.go index f1132155c4e2..5020b7931330 100644 --- a/pkg/util/constants.go +++ b/pkg/util/constants.go @@ -24,9 +24,8 @@ import ( // These constants are used by both minikube const ( - APIServerPort = 8443 - DefaultDNSDomain = "cluster.local" - DefaultServiceCIDR = "10.96.0.0/12" + APIServerPort = 8443 + DefaultDNSDomain = "cluster.local" ) // DefaultV114AdmissionControllers are admission controllers we default to in v1.14.x diff --git a/pkg/util/downloader.go b/pkg/util/downloader.go index 20ae0550ba53..72b478256589 100644 --- a/pkg/util/downloader.go +++ b/pkg/util/downloader.go @@ -20,13 +20,16 @@ import ( "net/url" "os" "path/filepath" + "time" "github.com/golang/glog" "github.com/hashicorp/go-getter" + "github.com/juju/mutex" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/util/lock" ) const fileScheme = "file" @@ -56,6 +59,18 @@ func (f DefaultDownloader) GetISOFileURI(isoURL string) string { // CacheMinikubeISOFromURL downloads the ISO, if it doesn't exist in cache func (f DefaultDownloader) CacheMinikubeISOFromURL(url string) error { + dst := f.GetISOCacheFilepath(url) + + // Lock before we check for existence to avoid thundering herd issues + spec := lock.PathMutexSpec(dst) + spec.Timeout = 10 * time.Minute + glog.Infof("acquiring lock: %+v", spec) + releaser, err := mutex.Acquire(spec) + if err != nil { + return errors.Wrapf(err, "unable to acquire lock for %+v", spec) + } + defer releaser.Release() + if !f.ShouldCacheMinikubeISO(url) { glog.Infof("Not caching ISO, using %s", url) return nil @@ -66,7 +81,6 @@ func (f DefaultDownloader) CacheMinikubeISOFromURL(url string) error { urlWithChecksum = url + "?checksum=file:" + constants.DefaultISOSHAURL } - dst := f.GetISOCacheFilepath(url) // Predictable temp destination so that resume can function tmpDst := dst + ".download" diff --git a/site/.gitignore b/site/.gitignore new file mode 100644 index 000000000000..b512c09d4766 --- /dev/null +++ b/site/.gitignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file diff --git a/site/config.toml b/site/config.toml index 53f04cb0ce09..190237e9d479 100644 --- a/site/config.toml +++ b/site/config.toml @@ -104,7 +104,7 @@ privacy_policy = "" version_menu = "Releases" # Repository configuration (URLs for in-page links to opening issues and suggesting changes) -github_repo = "http://github.com/kubernetes/minikube" +github_repo = "https://github.com/kubernetes/minikube" # An optional link to a related project repo. For example, the sibling repository where your product code lives. github_project_repo = "" diff --git a/site/content/en/docs/Contributing/documentation.en.md b/site/content/en/docs/Contributing/documentation.en.md index 56017f0484cd..b2cd2b997185 100644 --- a/site/content/en/docs/Contributing/documentation.en.md +++ b/site/content/en/docs/Contributing/documentation.en.md @@ -10,7 +10,7 @@ minikube's documentation is in [Markdown](https://www.markdownguide.org/cheat-sh * [Hugo](https://gohugo.io) * [Docsy](https://www.docsy.dev) -In production, the minikube website is served using [Netlify](http://netlify.com/) +In production, the minikube website is served using [Netlify](https://netlify.com/) ## Local documentation website diff --git a/site/content/en/docs/Contributing/guide.en.md b/site/content/en/docs/Contributing/guide.en.md index c7280098c5f6..b546dce37aab 100644 --- a/site/content/en/docs/Contributing/guide.en.md +++ b/site/content/en/docs/Contributing/guide.en.md @@ -13,7 +13,7 @@ description: > ### License Agreement -We'd love to accept your patches! Before we can take them, [please fill out either the individual or corporate Contributor License Agreement (CLA)](http://git.k8s.io/community/CLA.md) +We'd love to accept your patches! Before we can take them, [please fill out either the individual or corporate Contributor License Agreement (CLA)](https://git.k8s.io/community/CLA.md) ### Finding issues to work on @@ -33,7 +33,7 @@ Once you've discovered an issue to work on: 1. Submit an issue describing your proposed change 2. A reviewer will respond to your issue promptly. -3. If your proposed change is accepted, and you haven't already done so, sign the [Contributor License Agreement (CLA)](http://git.k8s.io/community/CLA.md) +3. If your proposed change is accepted, and you haven't already done so, sign the [Contributor License Agreement (CLA)](https://git.k8s.io/community/CLA.md) 4. Fork the minikube repository, develop and test your code changes. 5. Submit a pull request. diff --git a/site/content/en/docs/Tutorials/nvidia_gpu.md b/site/content/en/docs/Tutorials/nvidia_gpu.md index 3a11b7fec1dc..74aabee72a20 100644 --- a/site/content/en/docs/Tutorials/nvidia_gpu.md +++ b/site/content/en/docs/Tutorials/nvidia_gpu.md @@ -102,7 +102,7 @@ VM drivers supported by minikube for macOS doesn't support GPU passthrough: - [mist64/xhyve#108](https://github.com/mist64/xhyve/issues/108) - [moby/hyperkit#159](https://github.com/moby/hyperkit/issues/159) -- [VirtualBox docs](http://www.virtualbox.org/manual/ch09.html#pcipassthrough) +- [VirtualBox docs](https://www.virtualbox.org/manual/ch09.html#pcipassthrough) Also: @@ -119,7 +119,7 @@ Also: minikube supports Windows host through Hyper-V or VirtualBox. - VirtualBox doesn't support PCI passthrough for [Windows - host](http://www.virtualbox.org/manual/ch09.html#pcipassthrough). + host](https://www.virtualbox.org/manual/ch09.html#pcipassthrough). - Hyper-V supports DDA (discrete device assignment) but [only for Windows Server 2016](https://docs.microsoft.com/en-us/windows-server/virtualization/hyper-v/plan/plan-for-deploying-devices-using-discrete-device-assignment) diff --git a/test/integration/aab_offline_test.go b/test/integration/aab_offline_test.go index 4224ecda8dc1..a83d84b1b157 100644 --- a/test/integration/aab_offline_test.go +++ b/test/integration/aab_offline_test.go @@ -30,9 +30,9 @@ import ( func TestOffline(t *testing.T) { t.Run("group", func(t *testing.T) { for _, runtime := range []string{"docker", "crio", "containerd"} { + runtime := runtime t.Run(runtime, func(t *testing.T) { MaybeParallel(t) - WaitForStartSlot(t) if runtime != "docker" && NoneDriver() { t.Skipf("skipping %s - incompatible with none driver", t.Name()) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index f6f9cbad2e1d..86e49d716b43 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -36,8 +36,6 @@ import ( // TestAddons tests addons that require no special environment -- in parallel func TestAddons(t *testing.T) { - MaybeParallel(t) - WaitForStartSlot(t) profile := UniqueProfileName("addons") ctx, cancel := context.WithTimeout(context.Background(), 40*time.Minute) defer CleanupWithLogs(t, profile, cancel) @@ -95,7 +93,7 @@ func validateIngressAddon(ctx context.Context, t *testing.T, profile string) { if err := kapi.WaitForDeploymentToStabilize(client, "kube-system", "nginx-ingress-controller", 6*time.Minute); err != nil { t.Errorf("waiting for ingress-controller deployment to stabilize: %v", err) } - if _, err := PodWait(ctx, t, profile, "kube-system", "app.kubernetes.io/name=nginx-ingress-controller", 8*time.Minute); err != nil { + if _, err := PodWait(ctx, t, profile, "kube-system", "app.kubernetes.io/name=nginx-ingress-controller", 12*time.Minute); err != nil { t.Fatalf("wait: %v", err) } @@ -155,7 +153,7 @@ func validateRegistryAddon(ctx context.Context, t *testing.T, profile string) { if _, err := PodWait(ctx, t, profile, "kube-system", "actual-registry=true", 6*time.Minute); err != nil { t.Fatalf("wait: %v", err) } - if _, err := PodWait(ctx, t, profile, "kube-system", "registry-proxy=true", 6*time.Minute); err != nil { + if _, err := PodWait(ctx, t, profile, "kube-system", "registry-proxy=true", 10*time.Minute); err != nil { t.Fatalf("wait: %v", err) } diff --git a/test/integration/docker_test.go b/test/integration/docker_test.go index 6ed9f76202ee..c04ba8617d54 100644 --- a/test/integration/docker_test.go +++ b/test/integration/docker_test.go @@ -31,10 +31,9 @@ func TestDockerFlags(t *testing.T) { t.Skip("skipping: none driver does not support ssh or bundle docker") } MaybeParallel(t) - WaitForStartSlot(t) profile := UniqueProfileName("docker-flags") - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer CleanupWithLogs(t, profile, cancel) // Use the most verbose logging for the simplest test. If it fails, something is very wrong. diff --git a/test/integration/fn_tunnel_cmd.go b/test/integration/fn_tunnel_cmd.go index b9e43c8cc295..a080bc1b787c 100644 --- a/test/integration/fn_tunnel_cmd.go +++ b/test/integration/fn_tunnel_cmd.go @@ -71,7 +71,7 @@ func validateTunnelCmd(ctx context.Context, t *testing.T, profile string) { if err != nil { t.Fatalf("%s failed: %v", rr.Args, err) } - if _, err := PodWait(ctx, t, profile, "default", "run=nginx-svc", 2*time.Minute); err != nil { + if _, err := PodWait(ctx, t, profile, "default", "run=nginx-svc", 4*time.Minute); err != nil { t.Fatalf("wait: %v", err) } diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 1480538ca78e..1d20afa56e2e 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -92,6 +92,7 @@ func TestFunctional(t *testing.T) { {"ConfigCmd", validateConfigCmd}, {"DashboardCmd", validateDashboardCmd}, {"DNS", validateDNS}, + {"DryRun", validateDryRun}, {"StatusCmd", validateStatusCmd}, {"LogsCmd", validateLogsCmd}, {"MountCmd", validateMountCmd}, @@ -172,7 +173,7 @@ func validateKubectlGetPods(ctx context.Context, t *testing.T, profile string) { // validateAddonManager asserts that the kube-addon-manager pod is deployed properly func validateAddonManager(ctx context.Context, t *testing.T, profile string) { // If --wait=false, this may take a couple of minutes - if _, err := PodWait(ctx, t, profile, "kube-system", "component=kube-addon-manager", 5*time.Minute); err != nil { + if _, err := PodWait(ctx, t, profile, "kube-system", "component=kube-addon-manager", 10*time.Minute); err != nil { t.Fatalf("wait: %v", err) } } @@ -309,6 +310,32 @@ func validateDNS(ctx context.Context, t *testing.T, profile string) { } } +// validateDryRun asserts that the dry-run mode quickly exits with the right code +func validateDryRun(ctx context.Context, t *testing.T, profile string) { + // dry-run mode should always be able to finish quickly + mctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + // Too little memory! + startArgs := append([]string{"start", "-p", profile, "--dry-run", "--memory", "250MB"}, StartArgs()...) + c := exec.CommandContext(mctx, Target(), startArgs...) + rr, err := Run(t, c) + + wantCode := 78 // exit.Config + if rr.ExitCode != wantCode { + t.Errorf("dry-run(250MB) exit code = %d, wanted = %d: %v", rr.ExitCode, wantCode, err) + } + + dctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + startArgs = append([]string{"start", "-p", profile, "--dry-run"}, StartArgs()...) + c = exec.CommandContext(dctx, Target(), startArgs...) + rr, err = Run(t, c) + if rr.ExitCode != 0 || err != nil { + t.Errorf("dry-run exit code = %d, wanted = %d: %v", rr.ExitCode, 0, err) + } +} + // validateCacheCmd tests functionality of cache command (cache add, delete, list) func validateCacheCmd(ctx context.Context, t *testing.T, profile string) { if NoneDriver() { @@ -482,7 +509,7 @@ func validateServiceCmd(ctx context.Context, t *testing.T, profile string) { t.Logf("%s failed: %v (may not be an error)", rr.Args, err) } - if _, err := PodWait(ctx, t, profile, "default", "app=hello-node", 5*time.Minute); err != nil { + if _, err := PodWait(ctx, t, profile, "default", "app=hello-node", 10*time.Minute); err != nil { t.Fatalf("wait: %v", err) } @@ -548,46 +575,14 @@ func validateServiceCmd(ctx context.Context, t *testing.T, profile string) { // validateAddonsCmd asserts basic "addon" command functionality func validateAddonsCmd(ctx context.Context, t *testing.T, profile string) { - - // Default output + // Table output rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "list")) if err != nil { t.Errorf("%s failed: %v", rr.Args, err) } - listLines := strings.Split(strings.TrimSpace(rr.Stdout.String()), "\n") - r := regexp.MustCompile(`-\s[a-z|-]+:\s(enabled|disabled)`) - for _, line := range listLines { - match := r.MatchString(line) - if !match { - t.Errorf("Plugin output did not match expected format. Got: %s", line) - } - } - - // Custom format - rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "list", "--format", `"{{.AddonName}}":"{{.AddonStatus}}"`)) - if err != nil { - t.Errorf("%s failed: %v", rr.Args, err) - } - listLines = strings.Split(strings.TrimSpace(rr.Stdout.String()), "\n") - r = regexp.MustCompile(`"[a-z|-]+":"(enabled|disabled)"`) - for _, line := range listLines { - match := r.MatchString(line) - if !match { - t.Errorf("Plugin output did not match expected custom format. Got: %s", line) - } - } - - // Custom format shorthand - rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "list", "-f", `"{{.AddonName}}":"{{.AddonStatus}}"`)) - if err != nil { - t.Errorf("%s failed: %v", rr.Args, err) - } - listLines = strings.Split(strings.TrimSpace(rr.Stdout.String()), "\n") - r = regexp.MustCompile(`"[a-z|-]+":"(enabled|disabled)"`) - for _, line := range listLines { - match := r.MatchString(line) - if !match { - t.Errorf("Plugin output did not match expected custom format. Got: %s", line) + for _, a := range []string{"dashboard", "ingress", "ingress-dns"} { + if !strings.Contains(rr.Output(), a) { + t.Errorf("addon list expected to include %q but didn't output: %q", a, rr.Output()) } } @@ -625,7 +620,7 @@ func validateMySQL(ctx context.Context, t *testing.T, profile string) { t.Fatalf("%s failed: %v", rr.Args, err) } - names, err := PodWait(ctx, t, profile, "default", "app=mysql", 5*time.Minute) + names, err := PodWait(ctx, t, profile, "default", "app=mysql", 10*time.Minute) if err != nil { t.Fatalf("podwait: %v", err) } diff --git a/test/integration/guest_env_test.go b/test/integration/guest_env_test.go index 0e7dfb66347d..3182daf67a0e 100644 --- a/test/integration/guest_env_test.go +++ b/test/integration/guest_env_test.go @@ -28,7 +28,6 @@ import ( func TestGuestEnvironment(t *testing.T) { MaybeParallel(t) - WaitForStartSlot(t) profile := UniqueProfileName("guest") ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) diff --git a/test/integration/gvisor_addon_test.go b/test/integration/gvisor_addon_test.go index 172802ceea95..cd8051e4ef3b 100644 --- a/test/integration/gvisor_addon_test.go +++ b/test/integration/gvisor_addon_test.go @@ -35,7 +35,6 @@ func TestGvisorAddon(t *testing.T) { } MaybeParallel(t) - WaitForStartSlot(t) profile := UniqueProfileName("gvisor") ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) defer func() { @@ -49,7 +48,7 @@ func TestGvisorAddon(t *testing.T) { CleanupWithLogs(t, profile, cancel) }() - startArgs := append([]string{"start", "-p", profile, "--container-runtime=containerd", "--docker-opt", "containerd=/var/run/containerd/containerd.sock", "--wait=false"}, StartArgs()...) + startArgs := append([]string{"start", "-p", profile, "--container-runtime=containerd", "--docker-opt", "containerd=/var/run/containerd/containerd.sock"}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...)) if err != nil { t.Fatalf("%s failed: %v", rr.Args, err) @@ -61,20 +60,11 @@ func TestGvisorAddon(t *testing.T) { t.Logf("%s failed: %v (won't test local image)", rr.Args, err) } - // NOTE: addons are global, but the addon must assert that the runtime is containerd rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "gvisor")) if err != nil { t.Fatalf("%s failed: %v", rr.Args, err) } - // Because addons are persistent across profiles :( - defer func() { - rr, err := Run(t, exec.Command(Target(), "-p", profile, "addons", "disable", "gvisor")) - if err != nil { - t.Logf("%s failed: %v", rr.Args, err) - } - }() - if _, err := PodWait(ctx, t, profile, "kube-system", "kubernetes.io/minikube-addons=gvisor", 4*time.Minute); err != nil { t.Fatalf("waiting for gvisor controller to be up: %v", err) } diff --git a/test/integration/helpers.go b/test/integration/helpers.go index 1539170f7720..8d145da091bf 100644 --- a/test/integration/helpers.go +++ b/test/integration/helpers.go @@ -30,7 +30,6 @@ import ( "io/ioutil" "os/exec" "strings" - "sync" "testing" "time" @@ -41,13 +40,6 @@ import ( "k8s.io/minikube/pkg/kapi" ) -var ( - // startTimes is a list of startup times, to guarantee --start-offset - startTimes = []time.Time{} - // startTimesMutex is a lock to update startTimes without a race condition - startTimesMutex = &sync.Mutex{} -) - // RunResult stores the result of an cmd.Run call type RunResult struct { Stdout *bytes.Buffer @@ -349,34 +341,6 @@ func MaybeParallel(t *testing.T) { t.Parallel() } -// WaitForStartSlot enforces --start-offset to avoid startup race conditions -func WaitForStartSlot(t *testing.T) { - // Not parallel - if NoneDriver() { - return - } - - wakeup := time.Now() - startTimesMutex.Lock() - if len(startTimes) > 0 { - nextStart := startTimes[len(startTimes)-1].Add(*startOffset) - // Ignore nextStart if it is in the past - to guarantee offset for next caller - if time.Now().Before(nextStart) { - wakeup = nextStart - } - } - startTimes = append(startTimes, wakeup) - startTimesMutex.Unlock() - - if time.Now().Before(wakeup) { - d := time.Until(wakeup) - t.Logf("Waiting for start slot at %s (sleeping %s) ...", wakeup, d) - time.Sleep(d) - } else { - t.Logf("No need to wait for start slot, it is already %s", time.Now()) - } -} - // killProcessFamily kills a pid and all of its children func killProcessFamily(t *testing.T, pid int) { parent, err := process.NewProcess(int32(pid)) diff --git a/test/integration/main.go b/test/integration/main.go index fe351e070d46..bffb12f07966 100644 --- a/test/integration/main.go +++ b/test/integration/main.go @@ -33,7 +33,6 @@ var defaultDriver = flag.String("expected-default-driver", "", "Expected default var forceProfile = flag.String("profile", "", "force tests to run against a particular profile") var cleanup = flag.Bool("cleanup", true, "cleanup failed test run") var enableGvisor = flag.Bool("gvisor", false, "run gvisor integration test (slow)") -var startOffset = flag.Duration("start-offset", 30*time.Second, "how much time to offset between cluster starts") var postMortemLogs = flag.Bool("postmortem-logs", true, "show logs after a failed test run") // Paths to files - normally set for CI diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index 5b43864614cc..ad1b6832b5ff 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -78,7 +78,6 @@ func TestStartStop(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { MaybeParallel(t) - WaitForStartSlot(t) if !strings.Contains(tc.name, "docker") && NoneDriver() { t.Skipf("skipping %s - incompatible with none driver", t.Name()) @@ -107,7 +106,8 @@ func TestStartStop(t *testing.T) { t.Fatalf("%s failed: %v", rr.Args, err) } - names, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 4*time.Minute) + // 8 minutes, because 4 is not enough for images to pull in all cases. + names, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 8*time.Minute) if err != nil { t.Fatalf("wait: %v", err) } @@ -140,7 +140,6 @@ func TestStartStop(t *testing.T) { t.Errorf("status = %q; want = %q", got, state.Stopped) } - WaitForStartSlot(t) rr, err = Run(t, exec.CommandContext(ctx, Target(), startArgs...)) if err != nil { // Explicit fatal so that failures don't move directly to deletion @@ -163,10 +162,11 @@ func TestStartStop(t *testing.T) { gotImages := []string{} for _, img := range jv["images"] { for _, i := range img.Tags { - // Ignore non-Kubernetes images - if !strings.Contains(i, "ingress") && !strings.Contains(i, "busybox") { + if defaultImage(i) { // Remove docker.io for naming consistency between container runtimes gotImages = append(gotImages, strings.TrimPrefix(i, "docker.io/")) + } else { + t.Logf("Found non-minikube image: %s", i) } } } @@ -183,7 +183,7 @@ func TestStartStop(t *testing.T) { if strings.Contains(tc.name, "cni") { t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(") - } else if _, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 2*time.Minute); err != nil { + } else if _, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 4*time.Minute); err != nil { t.Fatalf("wait: %v", err) } @@ -203,3 +203,14 @@ func TestStartStop(t *testing.T) { } }) } + +// defaultImage returns true if this image is expected in a default minikube install +func defaultImage(name string) bool { + if strings.Contains(name, ":latest") { + return false + } + if strings.Contains(name, "k8s.gcr.io") || strings.Contains(name, "kubernetesui") || strings.Contains(name, "storage-provisioner") { + return true + } + return false +} diff --git a/test/integration/version_upgrade_test.go b/test/integration/version_upgrade_test.go index cec42dcb9d37..f4caae450a82 100644 --- a/test/integration/version_upgrade_test.go +++ b/test/integration/version_upgrade_test.go @@ -42,7 +42,6 @@ import ( // and it tries to upgrade from the older supported k8s to news supported k8s func TestVersionUpgrade(t *testing.T) { MaybeParallel(t) - WaitForStartSlot(t) profile := UniqueProfileName("vupgrade") ctx, cancel := context.WithTimeout(context.Background(), 55*time.Minute) @@ -92,7 +91,6 @@ func TestVersionUpgrade(t *testing.T) { t.Errorf("status = %q; want = %q", got, state.Stopped.String()) } - WaitForStartSlot(t) args = append([]string{"start", "-p", profile, fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion), "--alsologtostderr", "-v=1"}, StartArgs()...) rr, err = Run(t, exec.CommandContext(ctx, Target(), args...)) if err != nil {