From 0c68a59f076de60a8f9f2583b9ae6e57c924d57b Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Sun, 13 Dec 2020 18:59:19 -0800
Subject: [PATCH 01/22] kic-base-image-multiarch target

---
 Makefile | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/Makefile b/Makefile
index 5f2a7f35a9f5..094d41e96a6b 100644
--- a/Makefile
+++ b/Makefile
@@ -600,6 +600,24 @@ kic-base-image: ## builds the kic base image and tags local/kicbase:latest and l
 	docker tag local/kicbase:$(KIC_VERSION) local/kicbase:latest
 	docker tag local/kicbase:$(KIC_VERSION) local/kicbase:$(KIC_VERSION)-$(COMMIT_SHORT)
 
+X_DOCKER_BUILDER ?= minikube-builder
+X_BUILD_ENV ?= DOCKER_CLI_EXPERIMENTAL=enabled
+.PHONY: docker-multi-arch-builder
+docker-multi-arch-builder:
+	env $(X_BUILD_ENV) docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+	env $(X_BUILD_ENV) docker buildx create --name kicbase-builder --use || true
+
+KICBASE_ARCH = linux/arm64,linux/amd64
+X_IMG = gcr.io/kicbase/kicbase
+
+.PHONY: kic-base-image-x
+kic-base-image-x: docker-multi-arch-builder
+	env $(X_BUILD_ENV) docker buildx build --platform $(KICBASE_ARCH) -t $(X_IMG):$(KIC_VERSION) --push  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) ./deploy/kicbase
+
+
+
+
+
 .PHONY: upload-preloaded-images-tar
 upload-preloaded-images-tar: out/minikube # Upload the preloaded images for oldest supported, newest supported, and default kubernetes versions to GCS.
 	go build -ldflags="$(MINIKUBE_LDFLAGS)" -o out/upload-preload ./hack/preload-images/*.go

From 1860291cac1798fb4bb0140b1b028694031d48fb Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Sun, 13 Dec 2020 19:21:45 -0800
Subject: [PATCH 02/22] kic-base-image-multiarch target

---
 Makefile | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index 094d41e96a6b..915e01ffbec9 100644
--- a/Makefile
+++ b/Makefile
@@ -600,23 +600,24 @@ kic-base-image: ## builds the kic base image and tags local/kicbase:latest and l
 	docker tag local/kicbase:$(KIC_VERSION) local/kicbase:latest
 	docker tag local/kicbase:$(KIC_VERSION) local/kicbase:$(KIC_VERSION)-$(COMMIT_SHORT)
 
+# multi-arch docker images
 X_DOCKER_BUILDER ?= minikube-builder
 X_BUILD_ENV ?= DOCKER_CLI_EXPERIMENTAL=enabled
+
 .PHONY: docker-multi-arch-builder
 docker-multi-arch-builder:
 	env $(X_BUILD_ENV) docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 	env $(X_BUILD_ENV) docker buildx create --name kicbase-builder --use || true
 
 KICBASE_ARCH = linux/arm64,linux/amd64
-X_IMG = gcr.io/kicbase/kicbase
-
-.PHONY: kic-base-image-x
-kic-base-image-x: docker-multi-arch-builder
-	env $(X_BUILD_ENV) docker buildx build --platform $(KICBASE_ARCH) -t $(X_IMG):$(KIC_VERSION) --push  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) ./deploy/kicbase
-
-
-
-
+MA_KICBASE_IMAGE_REGISTRY = $(REGISTRY)/kicbase-multiarch:$(KIC_VERSION) $(REGISTRY_GH)/kicbase-multiarch:$(KIC_VERSION) kicbase/stable-multiarch:$(KIC_VERSION)
+
+.PHONY: kic-base-image-multi-arch
+kic-base-image-multi-arch: docker-multi-arch-builder
+#	docker login gcr.io/k8s-minikube
+#	docker login docker.pkg.github.com
+#	docker login
+	env $(X_BUILD_ENV) docker buildx build --platform $(KICBASE_ARCH) $(addprefix -t ,$(MA_KICBASE_IMAGE_REGISTRY)) --push  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) ./deploy/kicbase
 
 .PHONY: upload-preloaded-images-tar
 upload-preloaded-images-tar: out/minikube # Upload the preloaded images for oldest supported, newest supported, and default kubernetes versions to GCS.

From d1160305d23089c64042b412949dbdb5b733dd9e Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Sun, 13 Dec 2020 19:39:46 -0800
Subject: [PATCH 03/22] allow to override the list of ma-kicbase registries

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 915e01ffbec9..af141d7fa37d 100644
--- a/Makefile
+++ b/Makefile
@@ -610,7 +610,7 @@ docker-multi-arch-builder:
 	env $(X_BUILD_ENV) docker buildx create --name kicbase-builder --use || true
 
 KICBASE_ARCH = linux/arm64,linux/amd64
-MA_KICBASE_IMAGE_REGISTRY = $(REGISTRY)/kicbase-multiarch:$(KIC_VERSION) $(REGISTRY_GH)/kicbase-multiarch:$(KIC_VERSION) kicbase/stable-multiarch:$(KIC_VERSION)
+MA_KICBASE_IMAGE_REGISTRY ?= $(REGISTRY)/kicbase-multiarch:$(KIC_VERSION) $(REGISTRY_GH)/kicbase-multiarch:$(KIC_VERSION) kicbase/stable-multiarch:$(KIC_VERSION)
 
 .PHONY: kic-base-image-multi-arch
 kic-base-image-multi-arch: docker-multi-arch-builder

From 405ccc57dfbfbddf7ea5a24af70afd82a49e31d6 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Sun, 13 Dec 2020 21:10:51 -0800
Subject: [PATCH 04/22] skip docker login if AUTOPUSH is not defined

---
 Makefile | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index af141d7fa37d..770c9575f7c2 100644
--- a/Makefile
+++ b/Makefile
@@ -614,9 +614,11 @@ MA_KICBASE_IMAGE_REGISTRY ?= $(REGISTRY)/kicbase-multiarch:$(KIC_VERSION) $(REGI
 
 .PHONY: kic-base-image-multi-arch
 kic-base-image-multi-arch: docker-multi-arch-builder
-#	docker login gcr.io/k8s-minikube
-#	docker login docker.pkg.github.com
-#	docker login
+ifdef AUTOPUSH
+	docker login gcr.io/k8s-minikube
+	docker login docker.pkg.github.com
+	docker login
+endif
 	env $(X_BUILD_ENV) docker buildx build --platform $(KICBASE_ARCH) $(addprefix -t ,$(MA_KICBASE_IMAGE_REGISTRY)) --push  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) ./deploy/kicbase
 
 .PHONY: upload-preloaded-images-tar

From 6044006afee7eb8964c384a01087b3f1cc5efb4a Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Sun, 13 Dec 2020 22:29:17 -0800
Subject: [PATCH 05/22] add multi-arch images to kic driver list

---
 pkg/drivers/kic/types.go | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 32f314ba4a6a..757f8ab59983 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -26,15 +26,29 @@ const (
 	// Version is the current version of kic
 	Version = "v0.0.15-snapshot4"
 	// SHA of the kic base image
-	baseImageSHA = "ef1f485b5a1cfa4c989bc05e153f0a8525968ec999e242efff871cbb31649c16"
+	baseImageSHA          = "ef1f485b5a1cfa4c989bc05e153f0a8525968ec999e242efff871cbb31649c16"
+	baseMultiArchImageSHA = "todo"
 )
 
 var (
 	// BaseImage is the base image is used to spin up kic containers. it uses same base-image as kind.
-	BaseImage = fmt.Sprintf("gcr.io/k8s-minikube/kicbase:%s@sha256:%s", Version, baseImageSHA)
+	BaseImage = fmt.Sprintf("gcr.io/k8s-minikube/kicbase-multiarch:%s@sha256:%s", Version, baseMultiArchImageSHA)
 
 	// FallbackImages are backup base images in case gcr isn't available
 	FallbackImages = []string{
+
+		// the fallback of multi-arch BaseImage in case gcr.io is not available. stored in docker hub
+		// same image is push to https://github.com/kicbase/stable
+		fmt.Sprintf("kicbase/stable-multiarch:%s@sha256:%s", Version, baseMultiArchImageSHA),
+
+		// the fallback of multi-arch BaseImage in case gcr.io is not available. stored in github packages https://github.com/kubernetes/minikube/packages/206071
+		// github packages docker does _NOT_ support pulling by sha as mentioned in the docs:
+		// https://help.github.com/en/packages/using-github-packages-with-your-projects-ecosystem/configuring-docker-for-use-with-github-packages
+		fmt.Sprintf("docker.pkg.github.com/kubernetes/minikube/kicbase-multiarch:%s", Version),
+
+		// the amd64 only base image is used to spin up kic containers
+		fmt.Sprintf("gcr.io/k8s-minikube/kicbase:%s@sha256:%s", Version, baseImageSHA),
+
 		// the fallback of BaseImage in case gcr.io is not available. stored in docker hub
 		// same image is push to https://github.com/kicbase/stable
 		fmt.Sprintf("kicbase/stable:%s@sha256:%s", Version, baseImageSHA),

From 342b34c3392b512bf0c8b6479a87f6d9cd9369be Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Tue, 15 Dec 2020 11:20:20 -0800
Subject: [PATCH 06/22] fix unit tests

---
 site/content/en/docs/commands/start.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index b5bf841febdc..e92252237359 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.15-snapshot4@sha256:ef1f485b5a1cfa4c989bc05e153f0a8525968ec999e242efff871cbb31649c16")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-multiarch:v0.0.15-snapshot4@sha256:todo")
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cni string                        CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)
       --container-runtime string          The container runtime to be used (docker, cri-o, containerd). (default "docker")

From 63356581032dfe78f0bf48acd752cea934feeef3 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Tue, 5 Jan 2021 16:29:10 -0800
Subject: [PATCH 07/22] Revert single-arch kicbase fallback support

---
 Makefile                 |  2 +-
 pkg/drivers/kic/types.go | 15 ++-------------
 2 files changed, 3 insertions(+), 14 deletions(-)

diff --git a/Makefile b/Makefile
index 770c9575f7c2..f63abd40ad2c 100644
--- a/Makefile
+++ b/Makefile
@@ -610,7 +610,7 @@ docker-multi-arch-builder:
 	env $(X_BUILD_ENV) docker buildx create --name kicbase-builder --use || true
 
 KICBASE_ARCH = linux/arm64,linux/amd64
-MA_KICBASE_IMAGE_REGISTRY ?= $(REGISTRY)/kicbase-multiarch:$(KIC_VERSION) $(REGISTRY_GH)/kicbase-multiarch:$(KIC_VERSION) kicbase/stable-multiarch:$(KIC_VERSION)
+MA_KICBASE_IMAGE_REGISTRY ?= $(REGISTRY)/kicbase:$(KIC_VERSION) $(REGISTRY_GH)/kicbase-multiarch:$(KIC_VERSION) kicbase/stable-multiarch:$(KIC_VERSION)
 
 .PHONY: kic-base-image-multi-arch
 kic-base-image-multi-arch: docker-multi-arch-builder
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 757f8ab59983..af2b0689f468 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -26,26 +26,15 @@ const (
 	// Version is the current version of kic
 	Version = "v0.0.15-snapshot4"
 	// SHA of the kic base image
-	baseImageSHA          = "ef1f485b5a1cfa4c989bc05e153f0a8525968ec999e242efff871cbb31649c16"
-	baseMultiArchImageSHA = "todo"
+	baseImageSHA = "ef1f485b5a1cfa4c989bc05e153f0a8525968ec999e242efff871cbb31649c16"
 )
 
 var (
 	// BaseImage is the base image is used to spin up kic containers. it uses same base-image as kind.
-	BaseImage = fmt.Sprintf("gcr.io/k8s-minikube/kicbase-multiarch:%s@sha256:%s", Version, baseMultiArchImageSHA)
+	BaseImage = fmt.Sprintf("gcr.io/k8s-minikube/kicbase:%s@sha256:%s", Version, baseImageSHA)
 
 	// FallbackImages are backup base images in case gcr isn't available
 	FallbackImages = []string{
-
-		// the fallback of multi-arch BaseImage in case gcr.io is not available. stored in docker hub
-		// same image is push to https://github.com/kicbase/stable
-		fmt.Sprintf("kicbase/stable-multiarch:%s@sha256:%s", Version, baseMultiArchImageSHA),
-
-		// the fallback of multi-arch BaseImage in case gcr.io is not available. stored in github packages https://github.com/kubernetes/minikube/packages/206071
-		// github packages docker does _NOT_ support pulling by sha as mentioned in the docs:
-		// https://help.github.com/en/packages/using-github-packages-with-your-projects-ecosystem/configuring-docker-for-use-with-github-packages
-		fmt.Sprintf("docker.pkg.github.com/kubernetes/minikube/kicbase-multiarch:%s", Version),
-
 		// the amd64 only base image is used to spin up kic containers
 		fmt.Sprintf("gcr.io/k8s-minikube/kicbase:%s@sha256:%s", Version, baseImageSHA),
 

From df47190cca7938987d588334d4d5b7ae3556d82b Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Wed, 6 Jan 2021 11:54:22 -0800
Subject: [PATCH 08/22] fix docker builder creation args

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 55a6d60192c1..63b9fefbf6c8 100644
--- a/Makefile
+++ b/Makefile
@@ -607,7 +607,7 @@ X_BUILD_ENV ?= DOCKER_CLI_EXPERIMENTAL=enabled
 .PHONY: docker-multi-arch-builder
 docker-multi-arch-builder:
 	env $(X_BUILD_ENV) docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-	env $(X_BUILD_ENV) docker buildx create --name kicbase-builder --use || true
+	env $(X_BUILD_ENV) docker buildx create --node --name kicbase-builder --use || true
 
 KICBASE_ARCH = linux/arm64,linux/amd64
 MA_KICBASE_IMAGE_REGISTRY ?= $(REGISTRY)/kicbase:$(KIC_VERSION) $(REGISTRY_GH)/kicbase-multiarch:$(KIC_VERSION) kicbase/stable-multiarch:$(KIC_VERSION)

From 3aaf9c0f7c8092971f5acb0050daec04e6596e5d Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Fri, 8 Jan 2021 10:40:54 -0800
Subject: [PATCH 09/22] Add buildx debug flags

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 63b9fefbf6c8..935ce0467909 100644
--- a/Makefile
+++ b/Makefile
@@ -607,7 +607,7 @@ X_BUILD_ENV ?= DOCKER_CLI_EXPERIMENTAL=enabled
 .PHONY: docker-multi-arch-builder
 docker-multi-arch-builder:
 	env $(X_BUILD_ENV) docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-	env $(X_BUILD_ENV) docker buildx create --node --name kicbase-builder --use || true
+	env $(X_BUILD_ENV) docker buildx create --name kicbase-builder --buildkitd-flags '--debug' --use || true
 
 KICBASE_ARCH = linux/arm64,linux/amd64
 MA_KICBASE_IMAGE_REGISTRY ?= $(REGISTRY)/kicbase:$(KIC_VERSION) $(REGISTRY_GH)/kicbase-multiarch:$(KIC_VERSION) kicbase/stable-multiarch:$(KIC_VERSION)

From 1993e2eab622b9d950324c60960a3c7f1b711896 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Fri, 8 Jan 2021 10:57:08 -0800
Subject: [PATCH 10/22] fix kicbase image name

---
 Makefile                               | 2 +-
 site/content/en/docs/commands/start.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 935ce0467909..ee4c7dc5a96a 100644
--- a/Makefile
+++ b/Makefile
@@ -610,7 +610,7 @@ docker-multi-arch-builder:
 	env $(X_BUILD_ENV) docker buildx create --name kicbase-builder --buildkitd-flags '--debug' --use || true
 
 KICBASE_ARCH = linux/arm64,linux/amd64
-MA_KICBASE_IMAGE_REGISTRY ?= $(REGISTRY)/kicbase:$(KIC_VERSION) $(REGISTRY_GH)/kicbase-multiarch:$(KIC_VERSION) kicbase/stable-multiarch:$(KIC_VERSION)
+MA_KICBASE_IMAGE_REGISTRY ?= $(REGISTRY)/kicbase:$(KIC_VERSION) $(REGISTRY_GH)/kicbase:$(KIC_VERSION) kicbase/stable-multiarch:$(KIC_VERSION)
 
 .PHONY: kic-base-image-multi-arch
 kic-base-image-multi-arch: docker-multi-arch-builder
diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index d05d47888b6d..edaea7cb3a19 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-multiarch:v0.0.15-snapshot4@sha256:todo")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.15-snapshot4@sha256:ef1f485b5a1cfa4c989bc05e153f0a8525968ec999e242efff871cbb31649c16")
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cni string                        CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)
       --container-runtime string          The container runtime to be used (docker, cri-o, containerd). (default "docker")

From 9184efeb370e71040755405c394957bde4fc66a5 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Fri, 8 Jan 2021 11:06:37 -0800
Subject: [PATCH 11/22] fix Makefile rules

---
 Makefile | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index ee4c7dc5a96a..5f98ea01be39 100644
--- a/Makefile
+++ b/Makefile
@@ -610,16 +610,16 @@ docker-multi-arch-builder:
 	env $(X_BUILD_ENV) docker buildx create --name kicbase-builder --buildkitd-flags '--debug' --use || true
 
 KICBASE_ARCH = linux/arm64,linux/amd64
-MA_KICBASE_IMAGE_REGISTRY ?= $(REGISTRY)/kicbase:$(KIC_VERSION) $(REGISTRY_GH)/kicbase:$(KIC_VERSION) kicbase/stable-multiarch:$(KIC_VERSION)
+KICBASE_IMAGE_REGISTRIES ?= $(REGISTRY)/kicbase:$(KIC_VERSION) $(REGISTRY_GH)/kicbase:$(KIC_VERSION) kicbase/stable:$(KIC_VERSION)
 
-.PHONY: kic-base-image-multi-arch
-kic-base-image-multi-arch: docker-multi-arch-builder
+.PHONY: push-kic-base-image: 
+push-kic-base-image: docker-multi-arch-builder
 ifdef AUTOPUSH
 	docker login gcr.io/k8s-minikube
 	docker login docker.pkg.github.com
 	docker login
 endif
-	env $(X_BUILD_ENV) docker buildx build --platform $(KICBASE_ARCH) $(addprefix -t ,$(MA_KICBASE_IMAGE_REGISTRY)) --push  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) ./deploy/kicbase
+	env $(X_BUILD_ENV) docker buildx build --platform $(KICBASE_ARCH) $(addprefix -t ,$(KICBASE_IMAGE_REGISTRIES)) --push  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) ./deploy/kicbase
 
 .PHONY: upload-preloaded-images-tar
 upload-preloaded-images-tar: out/minikube # Upload the preloaded images for oldest supported, newest supported, and default kubernetes versions to GCS.
@@ -669,8 +669,8 @@ push-kic-base-image-hub: kic-base-image ## Push kic-base to docker hub
 	docker tag local/kicbase:latest $(KIC_BASE_IMAGE_HUB)
 	$(MAKE) push-docker IMAGE=$(KIC_BASE_IMAGE_HUB)
 
-.PHONY: push-kic-base-image
-push-kic-base-image: ## Push local/kicbase:latest to all remote registries
+.PHONY: push-kic-base-image-x86
+push-kic-base-image-x86: ## Push local/kicbase:latest to all remote registries
 ifndef AUTOPUSH
 	$(call user_confirm, 'Are you sure you want to push: $(KIC_BASE_IMAGE_GH) & $(KIC_BASE_IMAGE_GCR) & $(KIC_BASE_IMAGE_HUB) ?')
 	$(MAKE) push-kic-base-image AUTOPUSH=true

From da87abd016bfe410d26e7571ac5b33fc93f2c86f Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Fri, 8 Jan 2021 11:24:08 -0800
Subject: [PATCH 12/22] enable docker driver for arm64

---
 pkg/minikube/registry/drvs/docker/docker.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go
index 4a16ded7469d..9bfb61581ba0 100644
--- a/pkg/minikube/registry/drvs/docker/docker.go
+++ b/pkg/minikube/registry/drvs/docker/docker.go
@@ -84,7 +84,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
 }
 
 func status() registry.State {
-	if runtime.GOARCH != "amd64" {
+	if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" {
 		return registry.State{Error: fmt.Errorf("docker driver is not supported on %q systems yet", runtime.GOARCH), Installed: false, Healthy: false, Fix: "Try other drivers", Doc: docURL}
 	}
 

From f24a3e0a5149780a4d48a3c590f3bbe76b075306 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Fri, 8 Jan 2021 11:31:14 -0800
Subject: [PATCH 13/22] fix Makefile

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 5f98ea01be39..867e3f43ce1b 100644
--- a/Makefile
+++ b/Makefile
@@ -612,7 +612,7 @@ docker-multi-arch-builder:
 KICBASE_ARCH = linux/arm64,linux/amd64
 KICBASE_IMAGE_REGISTRIES ?= $(REGISTRY)/kicbase:$(KIC_VERSION) $(REGISTRY_GH)/kicbase:$(KIC_VERSION) kicbase/stable:$(KIC_VERSION)
 
-.PHONY: push-kic-base-image: 
+.PHONY: push-kic-base-image 
 push-kic-base-image: docker-multi-arch-builder
 ifdef AUTOPUSH
 	docker login gcr.io/k8s-minikube

From 6c860a48fe94cbfeb970297fabdb4137472d31e1 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Fri, 8 Jan 2021 11:49:10 -0800
Subject: [PATCH 14/22] confirm kicbase push

---
 Makefile | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Makefile b/Makefile
index 867e3f43ce1b..bd0b4b448682 100644
--- a/Makefile
+++ b/Makefile
@@ -607,6 +607,7 @@ X_BUILD_ENV ?= DOCKER_CLI_EXPERIMENTAL=enabled
 .PHONY: docker-multi-arch-builder
 docker-multi-arch-builder:
 	env $(X_BUILD_ENV) docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+	env $(X_BUILD_ENV) docker buildx rm --builder $(X_DOCKER_BUILDER) || true
 	env $(X_BUILD_ENV) docker buildx create --name kicbase-builder --buildkitd-flags '--debug' --use || true
 
 KICBASE_ARCH = linux/arm64,linux/amd64
@@ -618,6 +619,11 @@ ifdef AUTOPUSH
 	docker login gcr.io/k8s-minikube
 	docker login docker.pkg.github.com
 	docker login
+endif
+	$(foreach REG,$(KICBASE_IMAGE_REGISTRIES), \
+		@docker pull $(REG) && echo "Image already exist in registry" && exit 1 || echo "Image doesn't exist in registry";)
+ifndef AUTOPUSH
+	$(call user_confirm, 'Are you sure you want to push $(KICBASE_IMAGE_REGISTRIES) ?')
 endif
 	env $(X_BUILD_ENV) docker buildx build --platform $(KICBASE_ARCH) $(addprefix -t ,$(KICBASE_IMAGE_REGISTRIES)) --push  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) ./deploy/kicbase
 

From 921306d88a22a106f80b83b5396ee76d7970991c Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Fri, 8 Jan 2021 13:31:13 -0800
Subject: [PATCH 15/22] fix fallback kicbaseimages list

---
 pkg/drivers/kic/types.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 204422704da1..e060fec10b1c 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -35,9 +35,6 @@ var (
 
 	// FallbackImages are backup base images in case gcr isn't available
 	FallbackImages = []string{
-		// the amd64 only base image is used to spin up kic containers
-		fmt.Sprintf("gcr.io/k8s-minikube/kicbase:%s@sha256:%s", Version, baseImageSHA),
-
 		// the fallback of BaseImage in case gcr.io is not available. stored in docker hub
 		// same image is push to https://github.com/kicbase/stable
 		fmt.Sprintf("kicbase/stable:%s@sha256:%s", Version, baseImageSHA),

From 2622e9583c8480cee7265f00577e4313c4efe5e6 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Fri, 8 Jan 2021 13:49:27 -0800
Subject: [PATCH 16/22] bump kicbase version to v0.0.16-snapshot1

---
 pkg/drivers/kic/types.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index e060fec10b1c..597d83c3fc97 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -24,7 +24,7 @@ import (
 
 const (
 	// Version is the current version of kic
-	Version = "v0.0.15-snapshot4"
+	Version = "v0.0.16-snapshot1"
 	// SHA of the kic base image
 	baseImageSHA = "ef1f485b5a1cfa4c989bc05e153f0a8525968ec999e242efff871cbb31649c16"
 )

From 7ca15031e8e597dd787cd16ec8aac4e9e98b5b0b Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Mon, 11 Jan 2021 18:30:39 -0800
Subject: [PATCH 17/22] fix kicbase sha1

---
 pkg/drivers/kic/types.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 597d83c3fc97..f126f6fe8f4b 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -26,7 +26,7 @@ const (
 	// Version is the current version of kic
 	Version = "v0.0.16-snapshot1"
 	// SHA of the kic base image
-	baseImageSHA = "ef1f485b5a1cfa4c989bc05e153f0a8525968ec999e242efff871cbb31649c16"
+	baseImageSHA = "dff16232547bb3ac3f2a9e09a42246a96ecf8f40d9a1c5bcf5a37953690954b6"
 )
 
 var (

From d8866394797410f44663052083b82f0134031a9f Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Tue, 12 Jan 2021 11:31:09 -0800
Subject: [PATCH 18/22] fix unit tests

---
 site/content/en/docs/commands/start.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index aac90df39d8b..fe1ae8953982 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.15-snapshot4@sha256:ef1f485b5a1cfa4c989bc05e153f0a8525968ec999e242efff871cbb31649c16")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.16-snapshot1@sha256:dff16232547bb3ac3f2a9e09a42246a96ecf8f40d9a1c5bcf5a37953690954b6")
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cni string                        CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto)
       --container-runtime string          The container runtime to be used (docker, cri-o, containerd). (default "docker")

From 480c36d0c060f20712f57ddd7708d6ff467b6bba Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Tue, 12 Jan 2021 13:55:22 -0800
Subject: [PATCH 19/22] trigger build

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index bd0b4b448682..a23e06f47ade 100644
--- a/Makefile
+++ b/Makefile
@@ -845,3 +845,4 @@ else
 	 export UPDATE_TARGET="all" && \
 	 go run update_kubernetes_version.go)
 endif
+

From a13d15fe8592d6061cef54b3ddf74bcb0e1377c9 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Tue, 12 Jan 2021 15:56:07 -0800
Subject: [PATCH 20/22] remove old make target for pushing kicbase

---
 Makefile | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/Makefile b/Makefile
index a23e06f47ade..964b049e27a4 100644
--- a/Makefile
+++ b/Makefile
@@ -675,15 +675,6 @@ push-kic-base-image-hub: kic-base-image ## Push kic-base to docker hub
 	docker tag local/kicbase:latest $(KIC_BASE_IMAGE_HUB)
 	$(MAKE) push-docker IMAGE=$(KIC_BASE_IMAGE_HUB)
 
-.PHONY: push-kic-base-image-x86
-push-kic-base-image-x86: ## Push local/kicbase:latest to all remote registries
-ifndef AUTOPUSH
-	$(call user_confirm, 'Are you sure you want to push: $(KIC_BASE_IMAGE_GH) & $(KIC_BASE_IMAGE_GCR) & $(KIC_BASE_IMAGE_HUB) ?')
-	$(MAKE) push-kic-base-image AUTOPUSH=true
-else
-	$(MAKE) push-kic-base-image-gcr push-kic-base-image-hub push-kic-base-image-gh 
-endif
-
 .PHONY: out/gvisor-addon
 out/gvisor-addon: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go ## Build gvisor addon
 	$(if $(quiet),@echo "  GO       $@")

From 837cfb7aca14b02b3736d3926b31806b6640474e Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Tue, 12 Jan 2021 15:57:48 -0800
Subject: [PATCH 21/22] Revert "remove old make target for pushing kicbase"

This reverts commit a13d15fe8592d6061cef54b3ddf74bcb0e1377c9.
---
 Makefile | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/Makefile b/Makefile
index 964b049e27a4..a23e06f47ade 100644
--- a/Makefile
+++ b/Makefile
@@ -675,6 +675,15 @@ push-kic-base-image-hub: kic-base-image ## Push kic-base to docker hub
 	docker tag local/kicbase:latest $(KIC_BASE_IMAGE_HUB)
 	$(MAKE) push-docker IMAGE=$(KIC_BASE_IMAGE_HUB)
 
+.PHONY: push-kic-base-image-x86
+push-kic-base-image-x86: ## Push local/kicbase:latest to all remote registries
+ifndef AUTOPUSH
+	$(call user_confirm, 'Are you sure you want to push: $(KIC_BASE_IMAGE_GH) & $(KIC_BASE_IMAGE_GCR) & $(KIC_BASE_IMAGE_HUB) ?')
+	$(MAKE) push-kic-base-image AUTOPUSH=true
+else
+	$(MAKE) push-kic-base-image-gcr push-kic-base-image-hub push-kic-base-image-gh 
+endif
+
 .PHONY: out/gvisor-addon
 out/gvisor-addon: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go ## Build gvisor addon
 	$(if $(quiet),@echo "  GO       $@")

From 2ca3cd204f575e2e1855fbe46458f1bc2f2dfd27 Mon Sep 17 00:00:00 2001
From: Ilya Zuyev <izuyev@google.com>
Date: Tue, 12 Jan 2021 16:02:27 -0800
Subject: [PATCH 22/22] Fix Makefile

---
 Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index a23e06f47ade..7e4ca125769c 100644
--- a/Makefile
+++ b/Makefile
@@ -614,7 +614,7 @@ KICBASE_ARCH = linux/arm64,linux/amd64
 KICBASE_IMAGE_REGISTRIES ?= $(REGISTRY)/kicbase:$(KIC_VERSION) $(REGISTRY_GH)/kicbase:$(KIC_VERSION) kicbase/stable:$(KIC_VERSION)
 
 .PHONY: push-kic-base-image 
-push-kic-base-image: docker-multi-arch-builder
+push-kic-base-image: docker-multi-arch-builder ## Push multi-arch local/kicbase:latest to all remote registries
 ifdef AUTOPUSH
 	docker login gcr.io/k8s-minikube
 	docker login docker.pkg.github.com
@@ -675,8 +675,8 @@ push-kic-base-image-hub: kic-base-image ## Push kic-base to docker hub
 	docker tag local/kicbase:latest $(KIC_BASE_IMAGE_HUB)
 	$(MAKE) push-docker IMAGE=$(KIC_BASE_IMAGE_HUB)
 
-.PHONY: push-kic-base-image-x86
-push-kic-base-image-x86: ## Push local/kicbase:latest to all remote registries
+.PHONY: push-kic-base-image-x86-deprecated
+push-kic-base-image-x86-deprecated: ## Push legacy, non-multiarch local/kicbase:latest to all remote registries
 ifndef AUTOPUSH
 	$(call user_confirm, 'Are you sure you want to push: $(KIC_BASE_IMAGE_GH) & $(KIC_BASE_IMAGE_GCR) & $(KIC_BASE_IMAGE_HUB) ?')
 	$(MAKE) push-kic-base-image AUTOPUSH=true