From 62b2c94c0f557f7618287dd01901e455ef4ce0c4 Mon Sep 17 00:00:00 2001
From: Andriy Dzikh <andriydzikh@gmail.com>
Date: Thu, 13 May 2021 13:52:54 -0700
Subject: [PATCH 1/7] Refactor assets/addons.go to de-dupe map parsing.

---
 pkg/minikube/assets/addons.go | 59 ++++++++++++++++++++---------------
 1 file changed, 33 insertions(+), 26 deletions(-)

diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index d0e3acca6316..9df91919f2ac 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -660,6 +660,23 @@ var Addons = map[string]*Addon{
 	}),
 }
 
+// parseMapString creates a map based on `str` which is encoded as <key1>=<value1>,<key2>=<value2>,...
+func parseMapString(str string) map[string]string {
+	mapResult := make(map[string]string)
+	if str == "" {
+		return mapResult
+	}
+	for _, pairText := range strings.Split(str, ",") {
+		vals := strings.Split(pairText, "=")
+		if len(vals) != 2 {
+			out.WarningT("Ignoring invalid pair entry {{.pair}}", out.V{"pair": pairText})
+			continue
+		}
+		mapResult[vals[0]] = vals[1]
+	}
+	return mapResult
+}
+
 // GenerateTemplateData generates template data for template assets
 func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, netInfo NetworkInfo) interface{} {
 
@@ -705,19 +722,16 @@ func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, netInfo Net
 		opts.Images = make(map[string]string) // Avoid nil access when rendering
 	}
 
-	images := viper.GetString(config.AddonImages)
-	if images != "" {
-		for _, image := range strings.Split(images, ",") {
-			vals := strings.Split(image, "=")
-			if len(vals) != 2 || vals[1] == "" {
-				out.WarningT("Ignoring invalid custom image {{.conf}}", out.V{"conf": image})
-				continue
-			}
-			if _, ok := opts.Images[vals[0]]; ok {
-				opts.Images[vals[0]] = vals[1]
-			} else {
-				out.WarningT("Ignoring unknown custom image {{.name}}", out.V{"name": vals[0]})
-			}
+	images := parseMapString(viper.GetString(config.AddonImages))
+	for name, image := range images {
+		if image == "" {
+			out.WarningT("Ignoring empty custom image {{.name}}", out.V{"name": name})
+			continue
+		}
+		if _, ok := opts.Images[name]; ok {
+			opts.Images[name] = image
+		} else {
+			out.WarningT("Ignoring unknown custom image {{.name}}", out.V{"name": name})
 		}
 	}
 
@@ -725,19 +739,12 @@ func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, netInfo Net
 		opts.Registries = make(map[string]string)
 	}
 
-	registries := viper.GetString(config.AddonRegistries)
-	if registries != "" {
-		for _, registry := range strings.Split(registries, ",") {
-			vals := strings.Split(registry, "=")
-			if len(vals) != 2 {
-				out.WarningT("Ignoring invalid custom registry {{.conf}}", out.V{"conf": registry})
-				continue
-			}
-			if _, ok := opts.Images[vals[0]]; ok { // check images map because registry map may omitted default registry
-				opts.CustomRegistries[vals[0]] = vals[1]
-			} else {
-				out.WarningT("Ignoring unknown custom registry {{.name}}", out.V{"name": vals[0]})
-			}
+	registries := parseMapString(viper.GetString(config.AddonRegistries))
+	for name, registry := range registries {
+		if _, ok := opts.Images[name]; ok { // check images map because registry map may omitted default registry
+			opts.CustomRegistries[name] = registry
+		} else {
+			out.WarningT("Ignoring unknown custom registry {{.name}}", out.V{"name": registry})
 		}
 	}
 

From 7684a14e1a2d21d00b3b9226a919896369477fda Mon Sep 17 00:00:00 2001
From: Andriy Dzikh <andriydzikh@gmail.com>
Date: Fri, 14 May 2021 15:11:21 -0700
Subject: [PATCH 2/7] Move (custom) image selection to its own function.

---
 pkg/minikube/assets/addons.go | 94 +++++++++++++++++++++++------------
 1 file changed, 62 insertions(+), 32 deletions(-)

diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 9df91919f2ac..e81df9903dfd 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -677,6 +677,65 @@ func parseMapString(str string) map[string]string {
 	return mapResult
 }
 
+// mergeMaps creates a map with the union of `sourceMap` and `overrideMap` where collisions take the value of `overrideMap`.
+func mergeMaps(sourceMap, overrideMap map[string]string) map[string]string {
+	result := make(map[string]string)
+	for name, value := range sourceMap {
+		result[name] = value
+	}
+	for name, value := range overrideMap {
+		result[name] = value
+	}
+	return result
+}
+
+// filterKeySpace creates a map of the values in `targetMap` where the keys are also in `keySpace`.
+func filterKeySpace(keySpace map[string]string, targetMap map[string]string) map[string]string {
+	result := make(map[string]string)
+	for name := range keySpace {
+		if value, ok := targetMap[name]; ok {
+			result[name] = value
+		}
+	}
+	return result
+}
+
+// overrideDefaults creates a copy of `defaultMap` where `overrideMap` replaces any of its values that `overrideMap` contains.
+func overrideDefaults(defaultMap, overrideMap map[string]string) map[string]string {
+	return mergeMaps(defaultMap, filterKeySpace(defaultMap, overrideMap))
+}
+
+// selectImages uses AddonImages and AddonRegistries from viper to override default images and registries in `addon`.
+func selectImages(addon *Addon) (images, customRegistries map[string]string) {
+	addonDefaultImages := addon.Images
+	if addonDefaultImages == nil {
+		addonDefaultImages = make(map[string]string)
+	}
+
+	newImages := parseMapString(viper.GetString(config.AddonImages))
+	for name, image := range newImages {
+		if image == "" {
+			out.WarningT("Ignoring empty custom image {{.name}}", out.V{"name": name})
+			delete(newImages, name)
+			continue
+		}
+		if _, ok := addonDefaultImages[name]; !ok {
+			out.WarningT("Ignoring unknown custom image {{.name}}", out.V{"name": name})
+		}
+	}
+	// Use newly configured custom images.
+	images = overrideDefaults(addonDefaultImages, newImages)
+
+	customRegistries = parseMapString(viper.GetString(config.AddonRegistries))
+	for name := range customRegistries {
+		if _, ok := addonDefaultImages[name]; !ok { // check images map because registry map may omitted default registry
+			out.WarningT("Ignoring unknown custom registry {{.name}}", out.V{"name": name})
+			delete(customRegistries, name)
+		}
+	}
+	return images, customRegistries
+}
+
 // GenerateTemplateData generates template data for template assets
 func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, netInfo NetworkInfo) interface{} {
 
@@ -687,6 +746,7 @@ func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, netInfo Net
 	if runtime.GOARCH != "amd64" {
 		ea = "-" + runtime.GOARCH
 	}
+
 	opts := struct {
 		Arch                string
 		ExoticArch          string
@@ -705,8 +765,6 @@ func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, netInfo Net
 		LoadBalancerStartIP: cfg.LoadBalancerStartIP,
 		LoadBalancerEndIP:   cfg.LoadBalancerEndIP,
 		CustomIngressCert:   cfg.CustomIngressCert,
-		Images:              addon.Images,
-		Registries:          addon.Registries,
 		CustomRegistries:    make(map[string]string),
 		NetworkInfo:         make(map[string]string),
 	}
@@ -714,40 +772,12 @@ func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, netInfo Net
 		opts.ImageRepository += "/"
 	}
 
+	opts.Images, opts.CustomRegistries = selectImages(addon)
+
 	// Network info for generating template
 	opts.NetworkInfo["ControlPlaneNodeIP"] = netInfo.ControlPlaneNodeIP
 	opts.NetworkInfo["ControlPlaneNodePort"] = fmt.Sprint(netInfo.ControlPlaneNodePort)
 
-	if opts.Images == nil {
-		opts.Images = make(map[string]string) // Avoid nil access when rendering
-	}
-
-	images := parseMapString(viper.GetString(config.AddonImages))
-	for name, image := range images {
-		if image == "" {
-			out.WarningT("Ignoring empty custom image {{.name}}", out.V{"name": name})
-			continue
-		}
-		if _, ok := opts.Images[name]; ok {
-			opts.Images[name] = image
-		} else {
-			out.WarningT("Ignoring unknown custom image {{.name}}", out.V{"name": name})
-		}
-	}
-
-	if opts.Registries == nil {
-		opts.Registries = make(map[string]string)
-	}
-
-	registries := parseMapString(viper.GetString(config.AddonRegistries))
-	for name, registry := range registries {
-		if _, ok := opts.Images[name]; ok { // check images map because registry map may omitted default registry
-			opts.CustomRegistries[name] = registry
-		} else {
-			out.WarningT("Ignoring unknown custom registry {{.name}}", out.V{"name": registry})
-		}
-	}
-
 	// Append postfix "/" to registries
 	for k, v := range opts.Registries {
 		if v != "" && !strings.HasSuffix(v, "/") {

From 040e7e7a8d255801d14f076eca29f9dac73d4a62 Mon Sep 17 00:00:00 2001
From: Andriy Dzikh <andriydzikh@gmail.com>
Date: Fri, 14 May 2021 15:29:36 -0700
Subject: [PATCH 3/7] Persist custom addon image settings.

---
 pkg/addons/addons.go          |  8 +++-
 pkg/minikube/assets/addons.go | 75 ++++++++++++++++++++++++-----------
 pkg/minikube/config/types.go  |  2 +
 3 files changed, 61 insertions(+), 24 deletions(-)

diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go
index 251be26c8a8b..aa97479b4e55 100644
--- a/pkg/addons/addons.go
+++ b/pkg/addons/addons.go
@@ -185,6 +185,12 @@ https://github.com/kubernetes/minikube/issues/7332`, out.V{"driver_name": cc.Dri
 		exit.Error(reason.GuestCpConfig, "Error getting primary control plane", err)
 	}
 
+	// Persist images even if the machine is running so starting gets the correct images.
+	images, customRegistries, err := assets.SelectAndPersistImages(addon, cc)
+	if err != nil {
+		exit.Error(reason.HostSaveProfile, "Failed to persist images", err)
+	}
+
 	mName := config.MachineName(*cc, cp)
 	host, err := machine.LoadHost(api, mName)
 	if err != nil || !machine.IsRunning(api, mName) {
@@ -224,7 +230,7 @@ https://github.com/kubernetes/minikube/issues/7332`, out.V{"driver_name": cc.Dri
 		out.WarningT("At least needs control plane nodes to enable addon")
 	}
 
-	data := assets.GenerateTemplateData(addon, cc.KubernetesConfig, networkInfo)
+	data := assets.GenerateTemplateData(addon, cc.KubernetesConfig, networkInfo, images, customRegistries)
 	return enableOrDisableAddonInternal(cc, addon, runner, data, enable)
 }
 
diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index e81df9903dfd..779b5b6791dc 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -705,39 +705,65 @@ func overrideDefaults(defaultMap, overrideMap map[string]string) map[string]stri
 	return mergeMaps(defaultMap, filterKeySpace(defaultMap, overrideMap))
 }
 
-// selectImages uses AddonImages and AddonRegistries from viper to override default images and registries in `addon`.
-func selectImages(addon *Addon) (images, customRegistries map[string]string) {
+// SelectAndPersistImages selects which images to use based on addon default images, previously persisted images, and newly requested images - which are then persisted for future enables.
+func SelectAndPersistImages(addon *Addon, cc *config.ClusterConfig) (images, customRegistries map[string]string, err error) {
 	addonDefaultImages := addon.Images
 	if addonDefaultImages == nil {
 		addonDefaultImages = make(map[string]string)
 	}
 
-	newImages := parseMapString(viper.GetString(config.AddonImages))
-	for name, image := range newImages {
-		if image == "" {
-			out.WarningT("Ignoring empty custom image {{.name}}", out.V{"name": name})
-			delete(newImages, name)
-			continue
-		}
-		if _, ok := addonDefaultImages[name]; !ok {
-			out.WarningT("Ignoring unknown custom image {{.name}}", out.V{"name": name})
+	// Use previously configured custom images.
+	images = overrideDefaults(addonDefaultImages, cc.CustomAddonImages)
+	if viper.IsSet(config.AddonImages) {
+		// Parse the AddonImages flag if present.
+		newImages := parseMapString(viper.GetString(config.AddonImages))
+		for name, image := range newImages {
+			if image == "" {
+				out.WarningT("Ignoring empty custom image {{.name}}", out.V{"name": name})
+				delete(newImages, name)
+				continue
+			}
+			if _, ok := addonDefaultImages[name]; !ok {
+				out.WarningT("Ignoring unknown custom image {{.name}}", out.V{"name": name})
+			}
 		}
+		// Use newly configured custom images.
+		images = overrideDefaults(addonDefaultImages, newImages)
+		// Store custom addon images to be written.
+		cc.CustomAddonImages = mergeMaps(cc.CustomAddonImages, images)
 	}
-	// Use newly configured custom images.
-	images = overrideDefaults(addonDefaultImages, newImages)
 
-	customRegistries = parseMapString(viper.GetString(config.AddonRegistries))
-	for name := range customRegistries {
-		if _, ok := addonDefaultImages[name]; !ok { // check images map because registry map may omitted default registry
-			out.WarningT("Ignoring unknown custom registry {{.name}}", out.V{"name": name})
-			delete(customRegistries, name)
+	// Use previously configured custom registries.
+	customRegistries = filterKeySpace(addonDefaultImages, cc.CustomAddonRegistries) // filter by images map because registry map may omit default registry.
+	if viper.IsSet(config.AddonRegistries) {
+		// Parse the AddonRegistries flag if present.
+		customRegistries = parseMapString(viper.GetString(config.AddonRegistries))
+		for name := range customRegistries {
+			if _, ok := addonDefaultImages[name]; !ok { // check images map because registry map may omitted default registry
+				out.WarningT("Ignoring unknown custom registry {{.name}}", out.V{"name": name})
+				delete(customRegistries, name)
+			}
 		}
+		// Since registry map may omit default registry, any previously set custom registries for these images must be cleared.
+		for name := range addonDefaultImages {
+			delete(cc.CustomAddonRegistries, name)
+		}
+		// Merge newly set registries into custom addon registries to be written.
+		cc.CustomAddonRegistries = mergeMaps(cc.CustomAddonRegistries, customRegistries)
 	}
-	return images, customRegistries
+
+	err = nil
+	// If images or registries were specified, save the config afterward.
+	if viper.IsSet(config.AddonImages) || viper.IsSet(config.AddonRegistries) {
+		// Since these values are only set when a user enables an addon, it is safe to refer to the profile name.
+		err = config.Write(viper.GetString(config.ProfileName), cc)
+		// Whether err is nil or not we still return here.
+	}
+	return images, customRegistries, err
 }
 
 // GenerateTemplateData generates template data for template assets
-func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, netInfo NetworkInfo) interface{} {
+func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, netInfo NetworkInfo, images, customRegistries map[string]string) interface{} {
 
 	a := runtime.GOARCH
 	// Some legacy docker images still need the -arch suffix
@@ -765,14 +791,17 @@ func GenerateTemplateData(addon *Addon, cfg config.KubernetesConfig, netInfo Net
 		LoadBalancerStartIP: cfg.LoadBalancerStartIP,
 		LoadBalancerEndIP:   cfg.LoadBalancerEndIP,
 		CustomIngressCert:   cfg.CustomIngressCert,
-		CustomRegistries:    make(map[string]string),
+		Images:              images,
+		Registries:          addon.Registries,
+		CustomRegistries:    customRegistries,
 		NetworkInfo:         make(map[string]string),
 	}
 	if opts.ImageRepository != "" && !strings.HasSuffix(opts.ImageRepository, "/") {
 		opts.ImageRepository += "/"
 	}
-
-	opts.Images, opts.CustomRegistries = selectImages(addon)
+	if opts.Registries == nil {
+		opts.Registries = make(map[string]string)
+	}
 
 	// Network info for generating template
 	opts.NetworkInfo["ControlPlaneNodeIP"] = netInfo.ControlPlaneNodeIP
diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index 672d32bd1203..fb94d4b71c14 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -74,6 +74,8 @@ type ClusterConfig struct {
 	KubernetesConfig        KubernetesConfig
 	Nodes                   []Node
 	Addons                  map[string]bool
+	CustomAddonImages       map[string]string
+	CustomAddonRegistries   map[string]string
 	VerifyComponents        map[string]bool // map of components to verify and wait for after start.
 	StartHostTimeout        time.Duration
 	ScheduledStop           *ScheduledStopConfig

From 7998c8d691ad9e51f836d447c49b9c288f2b8d00 Mon Sep 17 00:00:00 2001
From: Andriy Dzikh <andriydzikh@gmail.com>
Date: Mon, 17 May 2021 11:43:53 -0700
Subject: [PATCH 4/7] Create tests to ensure custom addon images persist.

---
 test/integration/start_stop_delete_test.go | 40 ++++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)

diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index 471e85b1136f..90498f7be27b 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -107,6 +107,7 @@ func TestStartStop(t *testing.T) {
 					}{
 						{"FirstStart", validateFirstStart},
 						{"DeployApp", validateDeploying},
+						{"EnableAddonWhileActive", validateEnableAddonWhileActive},
 						{"Stop", validateStop},
 						{"EnableAddonAfterStop", validateEnableAddonAfterStop},
 						{"SecondStart", validateSecondStart},
@@ -169,6 +170,30 @@ func validateDeploying(ctx context.Context, t *testing.T, profile string, tcName
 	}
 }
 
+func validateEnableAddonWhileActive(ctx context.Context, t *testing.T, profile string, tcName string, tcVersion string, startArgs []string) {
+	defer PostMortemLogs(t, profile)
+
+	// Enable an addon to assert it requests the correct image.
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "enable", "metrics-server", "-p", profile, "--images=MetricsServer=k8s.gcr.io/echoserver:1.4", "--registries=MetricsServer=fake.domain"))
+	if err != nil {
+		t.Errorf("failed to enable an addon post-stop. args %q: %v", rr.Command(), err)
+	}
+
+	if strings.Contains(tcName, "cni") {
+		t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(")
+		return
+	}
+
+	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "describe", "deploy/metrics-server", "-n", "kube-system"))
+	if err != nil {
+		t.Errorf("failed to get info on auto-pause deployments. args %q: %v", rr.Command(), err)
+	}
+	deploymentInfo := rr.Stdout.String()
+	if !strings.Contains(deploymentInfo, " fake.domain/k8s.gcr.io/echoserver:1.4") {
+		t.Errorf("addon did not load correct image. Expected to contain \" fake.domain/k8s.gcr.io/echoserver:1.4\". Addon deployment info: %s", deploymentInfo)
+	}
+}
+
 // validateStop tests minikube stop
 func validateStop(ctx context.Context, t *testing.T, profile string, tcName string, tcVersion string, startArgs []string) {
 	defer PostMortemLogs(t, profile)
@@ -190,7 +215,7 @@ func validateEnableAddonAfterStop(ctx context.Context, t *testing.T, profile str
 	}
 
 	// Enable an addon to assert it comes up afterwards
-	rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "enable", "dashboard", "-p", profile))
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "enable", "dashboard", "-p", profile, "--images=MetricsScraper=k8s.gcr.io/echoserver:1.4"))
 	if err != nil {
 		t.Errorf("failed to enable an addon post-stop. args %q: %v", rr.Command(), err)
 	}
@@ -229,9 +254,20 @@ func validateAddonAfterStop(ctx context.Context, t *testing.T, profile string, t
 	defer PostMortemLogs(t, profile)
 	if strings.Contains(tcName, "cni") {
 		t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(")
-	} else if _, err := PodWait(ctx, t, profile, "kubernetes-dashboard", "k8s-app=kubernetes-dashboard", Minutes(9)); err != nil {
+		return
+	}
+	if _, err := PodWait(ctx, t, profile, "kubernetes-dashboard", "k8s-app=kubernetes-dashboard", Minutes(9)); err != nil {
 		t.Errorf("failed waiting for 'addon dashboard' pod post-stop-start: %v", err)
 	}
+
+	rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "describe", "deploy/dashboard-metrics-scraper", "-n", "kubernetes-dashboard"))
+	if err != nil {
+		t.Errorf("failed to get info on kubernetes-dashboard deployments. args %q: %v", rr.Command(), err)
+	}
+	deploymentInfo := rr.Stdout.String()
+	if !strings.Contains(deploymentInfo, " k8s.gcr.io/echoserver:1.4") {
+		t.Errorf("addon did not load correct image. Expected to contain \" k8s.gcr.io/echoserver:1.4\". Addon deployment info: %s", deploymentInfo)
+	}
 }
 
 // validateKubernetesImages verifies that a restarted cluster contains all the necessary images

From 46884123a8a02439f7d8ba2ebf26fefd8c34253b Mon Sep 17 00:00:00 2001
From: Andriy Dzikh <andriydzikh@gmail.com>
Date: Tue, 18 May 2021 10:27:53 -0700
Subject: [PATCH 5/7] Run make generate-docs.

---
 site/content/en/docs/contrib/tests.en.md   | 3 +++
 test/integration/start_stop_delete_test.go | 1 +
 translations/de.json                       | 5 +++--
 translations/es.json                       | 5 +++--
 translations/fr.json                       | 5 +++--
 translations/ja.json                       | 5 +++--
 translations/ko.json                       | 5 +++--
 translations/pl.json                       | 5 +++--
 translations/strings.txt                   | 5 +++--
 translations/zh-CN.json                    | 5 +++--
 10 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/site/content/en/docs/contrib/tests.en.md b/site/content/en/docs/contrib/tests.en.md
index 1e43bcd315e2..ec605bbd23cb 100644
--- a/site/content/en/docs/contrib/tests.en.md
+++ b/site/content/en/docs/contrib/tests.en.md
@@ -332,6 +332,9 @@ runs the initial minikube start
 #### validateDeploying
 deploys an app the minikube cluster
 
+#### validateEnableAddonWhileActive
+makes sure addons can be enabled while cluster is active.
+
 #### validateStop
 tests minikube stop
 
diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go
index 90498f7be27b..85e0f0f924cc 100644
--- a/test/integration/start_stop_delete_test.go
+++ b/test/integration/start_stop_delete_test.go
@@ -170,6 +170,7 @@ func validateDeploying(ctx context.Context, t *testing.T, profile string, tcName
 	}
 }
 
+// validateEnableAddonWhileActive makes sure addons can be enabled while cluster is active.
 func validateEnableAddonWhileActive(ctx context.Context, t *testing.T, profile string, tcName string, tcVersion string, startArgs []string) {
 	defer PostMortemLogs(t, profile)
 
diff --git a/translations/de.json b/translations/de.json
index 17cea8384a36..fac162380c7b 100644
--- a/translations/de.json
+++ b/translations/de.json
@@ -241,6 +241,7 @@
 	"Failed to list cached images": "",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "",
 	"Failed to remove image": "",
@@ -314,8 +315,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "",
diff --git a/translations/es.json b/translations/es.json
index 456bd63922b9..3a5385341fa2 100644
--- a/translations/es.json
+++ b/translations/es.json
@@ -246,6 +246,7 @@
 	"Failed to list cached images": "",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "",
 	"Failed to remove image": "",
@@ -319,8 +320,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "",
diff --git a/translations/fr.json b/translations/fr.json
index a8d5ed46f653..795b3ac85f4d 100644
--- a/translations/fr.json
+++ b/translations/fr.json
@@ -243,6 +243,7 @@
 	"Failed to list cached images": "",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "",
 	"Failed to remove image": "",
@@ -316,8 +317,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "",
diff --git a/translations/ja.json b/translations/ja.json
index 03ef3465e601..e135ea5b7fd3 100644
--- a/translations/ja.json
+++ b/translations/ja.json
@@ -234,6 +234,7 @@
 	"Failed to list cached images": "",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "",
 	"Failed to remove image": "",
@@ -304,8 +305,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "イメージ用コマンド:",
diff --git a/translations/ko.json b/translations/ko.json
index bd237a8bd122..919ac1b60fb3 100644
--- a/translations/ko.json
+++ b/translations/ko.json
@@ -263,6 +263,7 @@
 	"Failed to list cached images": "캐시된 이미지를 조회하는 데 실패하였습니다",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "캐시된 이미지를 다시 불러오는 데 실패하였습니다",
 	"Failed to remove image": "",
@@ -338,8 +339,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "이미지 명령어",
diff --git a/translations/pl.json b/translations/pl.json
index 34d0ba69cf8b..223afa20589e 100644
--- a/translations/pl.json
+++ b/translations/pl.json
@@ -251,6 +251,7 @@
 	"Failed to list cached images": "",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "",
 	"Failed to remove image": "",
@@ -326,8 +327,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "",
diff --git a/translations/strings.txt b/translations/strings.txt
index abf26870a45c..36aa212d4f44 100644
--- a/translations/strings.txt
+++ b/translations/strings.txt
@@ -226,6 +226,7 @@
 	"Failed to list cached images": "",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "",
 	"Failed to remove image": "",
@@ -294,8 +295,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "",
diff --git a/translations/zh-CN.json b/translations/zh-CN.json
index 852adfc7d1f2..64c27b87980a 100644
--- a/translations/zh-CN.json
+++ b/translations/zh-CN.json
@@ -312,6 +312,7 @@
 	"Failed to list cached images": "无法列出缓存镜像",
 	"Failed to list images": "",
 	"Failed to load image": "",
+	"Failed to persist images": "",
 	"Failed to pull image": "",
 	"Failed to reload cached images": "重新加载缓存镜像失败",
 	"Failed to remove image": "",
@@ -395,8 +396,8 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
-	"Ignoring invalid custom image {{.conf}}": "",
-	"Ignoring invalid custom registry {{.conf}}": "",
+	"Ignoring empty custom image {{.name}}": "",
+	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",
 	"Ignoring unknown custom registry {{.name}}": "",
 	"Images Commands:": "",

From 6c3113706e1d1484eb710f32af7fba4127509fb0 Mon Sep 17 00:00:00 2001
From: Andriy Dzikh <andriydzikh@gmail.com>
Date: Tue, 18 May 2021 13:28:56 -0700
Subject: [PATCH 6/7] Create unit tests for new utility functions in
 pkg/minikube/assets/addons.go.

---
 pkg/minikube/assets/addons_test.go | 144 +++++++++++++++++++++++++++++
 1 file changed, 144 insertions(+)
 create mode 100644 pkg/minikube/assets/addons_test.go

diff --git a/pkg/minikube/assets/addons_test.go b/pkg/minikube/assets/addons_test.go
new file mode 100644
index 000000000000..22530cff069c
--- /dev/null
+++ b/pkg/minikube/assets/addons_test.go
@@ -0,0 +1,144 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package assets
+
+import "testing"
+
+// mapsEqual returns true if and only if `a` contains all the same pairs as `b`.
+func mapsEqual(a, b map[string]string) bool {
+	for aKey, aValue := range a {
+		if bValue, ok := b[aKey]; !ok || aValue != bValue {
+			return false
+		}
+	}
+
+	for bKey := range b {
+		if _, ok := a[bKey]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+func TestParseMapString(t *testing.T) {
+	cases := map[string]map[string]string{
+		"Ardvark=1,B=2,Cantaloupe=3":         {"Ardvark": "1", "B": "2", "Cantaloupe": "3"},
+		"A=,B=2,C=":                          {"A": "", "B": "2", "C": ""},
+		"":                                   {},
+		"malformed,good=howdy,manyequals==,": {"good": "howdy"},
+	}
+	for actual, expected := range cases {
+		if parsedMap := parseMapString(actual); !mapsEqual(parsedMap, expected) {
+			t.Errorf("Parsed map from string \"%s\" differs from expected map: Actual: %v Expected: %v", actual, parsedMap, expected)
+		}
+	}
+}
+
+func TestMergeMaps(t *testing.T) {
+	type TestCase struct {
+		sourceMap   map[string]string
+		overrideMap map[string]string
+		expectedMap map[string]string
+	}
+	cases := []TestCase{
+		{
+			sourceMap:   map[string]string{"A": "1", "B": "2"},
+			overrideMap: map[string]string{"B": "7", "C": "3"},
+			expectedMap: map[string]string{"A": "1", "B": "7", "C": "3"},
+		},
+		{
+			sourceMap:   map[string]string{"B": "7", "C": "3"},
+			overrideMap: map[string]string{"A": "1", "B": "2"},
+			expectedMap: map[string]string{"A": "1", "B": "2", "C": "3"},
+		},
+		{
+			sourceMap:   map[string]string{"B": "7", "C": "3"},
+			overrideMap: map[string]string{},
+			expectedMap: map[string]string{"B": "7", "C": "3"},
+		},
+		{
+			sourceMap:   map[string]string{},
+			overrideMap: map[string]string{"B": "7", "C": "3"},
+			expectedMap: map[string]string{"B": "7", "C": "3"},
+		},
+	}
+	for _, test := range cases {
+		if actualMap := mergeMaps(test.sourceMap, test.overrideMap); !mapsEqual(actualMap, test.expectedMap) {
+			t.Errorf("Merging maps (source=%v, override=%v) differs from expected map: Actual: %v Expected: %v", test.sourceMap, test.overrideMap, actualMap, test.expectedMap)
+		}
+	}
+}
+
+func TestFilterKeySpace(t *testing.T) {
+	type TestCase struct {
+		keySpace    map[string]string
+		targetMap   map[string]string
+		expectedMap map[string]string
+	}
+	cases := []TestCase{
+		{
+			keySpace:    map[string]string{"A": "0", "B": ""},
+			targetMap:   map[string]string{"B": "1", "C": "2", "D": "3"},
+			expectedMap: map[string]string{"B": "1"},
+		},
+		{
+			keySpace:    map[string]string{},
+			targetMap:   map[string]string{"B": "1", "C": "2", "D": "3"},
+			expectedMap: map[string]string{},
+		},
+		{
+			keySpace:    map[string]string{"B": "1", "C": "2", "D": "3"},
+			targetMap:   map[string]string{},
+			expectedMap: map[string]string{},
+		},
+	}
+	for _, test := range cases {
+		if actualMap := filterKeySpace(test.keySpace, test.targetMap); !mapsEqual(actualMap, test.expectedMap) {
+			t.Errorf("Filtering keyspace of map (keyspace=%v, target=%v) differs from expected map: Actual: %v Expected: %v", test.keySpace, test.targetMap, actualMap, test.expectedMap)
+		}
+	}
+}
+
+func TestOverrideDefautls(t *testing.T) {
+	type TestCase struct {
+		defaultMap  map[string]string
+		overrideMap map[string]string
+		expectedMap map[string]string
+	}
+	cases := []TestCase{
+		{
+			defaultMap:  map[string]string{"A": "1", "B": "2", "C": "3"},
+			overrideMap: map[string]string{"B": "7", "C": "8"},
+			expectedMap: map[string]string{"A": "1", "B": "7", "C": "8"},
+		},
+		{
+			defaultMap:  map[string]string{"A": "1", "B": "2", "C": "3"},
+			overrideMap: map[string]string{"B": "7", "D": "8", "E": "9"},
+			expectedMap: map[string]string{"A": "1", "B": "7", "C": "3"},
+		},
+		{
+			defaultMap:  map[string]string{"A": "1", "B": "2", "C": "3"},
+			overrideMap: map[string]string{"B": "7", "D": "8", "E": "9"},
+			expectedMap: map[string]string{"A": "1", "B": "7", "C": "3"},
+		},
+	}
+	for _, test := range cases {
+		if actualMap := overrideDefaults(test.defaultMap, test.overrideMap); !mapsEqual(actualMap, test.expectedMap) {
+			t.Errorf("Override defaults (defaults=%v, overrides=%v) differs from expected map: Actual: %v Expected: %v", test.defaultMap, test.overrideMap, actualMap, test.expectedMap)
+		}
+	}
+}

From df32362a7f8acdfd3ebac16e494327d1e92f9aae Mon Sep 17 00:00:00 2001
From: Andriy Dzikh <andriydzikh@gmail.com>
Date: Wed, 19 May 2021 15:26:28 -0700
Subject: [PATCH 7/7] Add comments to types.go for
 CustomAddonImages/CustomAddonRegistries.

---
 pkg/minikube/config/types.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index fb94d4b71c14..aee9b1ac2840 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -74,9 +74,9 @@ type ClusterConfig struct {
 	KubernetesConfig        KubernetesConfig
 	Nodes                   []Node
 	Addons                  map[string]bool
-	CustomAddonImages       map[string]string
-	CustomAddonRegistries   map[string]string
-	VerifyComponents        map[string]bool // map of components to verify and wait for after start.
+	CustomAddonImages       map[string]string // Maps image names to the image to use for addons. e.g. Dashboard -> k8s.gcr.io/echoserver:1.4 makes dashboard addon use echoserver for its Dashboard deployment.
+	CustomAddonRegistries   map[string]string // Maps image names to the registry to use for addons. See CustomAddonImages for example.
+	VerifyComponents        map[string]bool   // map of components to verify and wait for after start.
 	StartHostTimeout        time.Duration
 	ScheduledStop           *ScheduledStopConfig
 	ExposedPorts            []string // Only used by the docker and podman driver