From 3672753100f9f7b68f1cb67d51b851c4043a836d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Tue, 10 Nov 2020 16:54:39 +0100
Subject: [PATCH 01/45] Change back to the version-pinned cri-o channel

---
 deploy/kicbase/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile
index 66e6fd7bbe7a..4fef0263aff4 100644
--- a/deploy/kicbase/Dockerfile
+++ b/deploy/kicbase/Dockerfile
@@ -113,8 +113,8 @@ RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/li
     clean-install containers-common catatonit conmon containernetworking-plugins cri-tools podman-plugins varlink
 
 # install cri-o based on https://github.com/cri-o/cri-o/blob/release-1.18/README.md#installing-cri-o
-RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:1.18.list" && \
-    curl -LO https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18/xUbuntu_20.04/Release.key && \
+RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18:/1.18.4/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:1.18.list" && \
+    curl -LO https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.18:/1.18.4/xUbuntu_20.04/Release.key && \
     apt-key add - < Release.key && \
     clean-install cri-o cri-o-runc
 

From b5e1d7ef17dd36a533d17f1f1ec53f1916221a99 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Fri, 20 Nov 2020 14:27:55 -0800
Subject: [PATCH 02/45] metrics

---
 hack/metrics/README.md   |   4 ++
 hack/metrics/metrics.go  | 138 +++++++++++++++++++++++++++++++++++++++
 hack/metrics/minikube.go |  26 ++++++++
 3 files changed, 168 insertions(+)
 create mode 100644 hack/metrics/README.md
 create mode 100644 hack/metrics/metrics.go
 create mode 100644 hack/metrics/minikube.go

diff --git a/hack/metrics/README.md b/hack/metrics/README.md
new file mode 100644
index 000000000000..ab82671fc6f2
--- /dev/null
+++ b/hack/metrics/README.md
@@ -0,0 +1,4 @@
+This script runs `minikube start` in a loop times how long it takes.
+It exports this data to Stackdriver via the OpenTelemetry API.
+
+This script is used to track minikube performance and prevent regressions.
diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go
new file mode 100644
index 000000000000..8a7582cada9a
--- /dev/null
+++ b/hack/metrics/metrics.go
@@ -0,0 +1,138 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"time"
+
+	_ "cloud.google.com/go/storage"
+	"contrib.go.opencensus.io/exporter/stackdriver"
+	"github.com/pkg/errors"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+	"go.opencensus.io/trace"
+)
+
+const (
+	projectEnvVar    = "MINIKUBE_GCP_PROJECT_ID"
+	customMetricName = "custom.googleapis.com/minikube/start_time"
+	profile          = "cloud-monitoring"
+)
+
+var (
+	// The task latency in milliseconds.
+	latencyS = stats.Float64("repl/start_time", "start time in seconds", "s")
+)
+
+func main() {
+	if err := execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func execute() error {
+	projectID := os.Getenv(projectEnvVar)
+	if projectID == "" {
+		return nil, fmt.Errorf("GCP tracer requires a valid GCP project id set via the %s env variable", projectEnvVar)
+	}
+
+	osMethod, err := tag.NewKey("os")
+	if err != nil {
+		return errors.Wrap(err, "new tag key")
+	}
+
+	ctx, err := tag.New(context.Background(), tag.Insert(osMethod, runtime.GOOS), tag.Insert(driverMethod, "docker"))
+	if err != nil {
+		return errors.Wrap(err, "new tag")
+	}
+	// Register the view. It is imperative that this step exists,
+	// otherwise recorded metrics will be dropped and never exported.
+	v := &view.View{
+		Name:        customMetricName,
+		Measure:     latencyS,
+		Description: "minikube start time",
+		Aggregation: view.LastValue(),
+	}
+	if err := view.Register(v); err != nil {
+		log.Fatalf("Failed to register the view: %v", err)
+	}
+
+	sd, err := stackdriver.NewExporter(stackdriver.Options{
+		ProjectID: projectID,
+		// MetricPrefix helps uniquely identify your metrics.
+		MetricPrefix: "minikube_start",
+		// ReportingInterval sets the frequency of reporting metrics
+		// to stackdriver backend.
+		ReportingInterval: 1 * time.Second,
+	})
+	if err != nil {
+		return errors.Wrap(err, "getting stackdriver exporter")
+	}
+	// It is imperative to invoke flush before your main function exits
+	defer sd.Flush()
+
+	// Register it as a trace exporter
+	trace.RegisterExporter(sd)
+
+	if err := sd.StartMetricsExporter(); err != nil {
+		log.Fatalf("Error starting metric exporter: %v", err)
+	}
+	defer sd.StopMetricsExporter()
+
+	// Record 2p0 start time values
+	for {
+		st := minikubeStartTime(ctx)
+		fmt.Printf("Latency: %f\n", st)
+		stats.Record(ctx, latencyS.M(st))
+		time.Sleep(2 * time.Second)
+	}
+}
+
+func minikubeStartTime(ctx context.Context) (float64, error) {
+	defer deleteMinikube(ctx, profile)
+
+	minikube := filepath.Join(os.Getenv("HOME"), "minikube/out/minikube")
+
+	cmd := exec.CommandContext(ctx, minikube, "start", "--driver=docker", "-p", profile)
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+
+	t := time.Now()
+	log.Print("Running minikube start....")
+	if err := cmd.Run(); err != nil {
+		log.Fatal(err)
+	}
+	return time.Since(t).Seconds()
+}
+
+func deleteMinikube(ctx context.Context) {
+	cmd := exec.CommandContext(ctx, minikube, "delete", "-p", profile)
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		log.Printf("error deleting: %v", err)
+	}
+}
diff --git a/hack/metrics/minikube.go b/hack/metrics/minikube.go
new file mode 100644
index 000000000000..712a8dd51636
--- /dev/null
+++ b/hack/metrics/minikube.go
@@ -0,0 +1,26 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+const (
+	repo = "https://github.com/kubernetes/minikube"
+)
+
+// buildMinikubeAtHEAD copies minikube to a temp dir
+func buildMinikubeAtHEAD() error {
+
+}

From fee0f5310a1febf984799ed59af1c86883fb75a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Mon, 16 Nov 2020 17:40:57 +0100
Subject: [PATCH 03/45] Add private network implementation for podman

Most of it the same as docker, except for the options.

i.e. libnetwork "bridge" plugin vs. cni "bridge" plugin
---
 cmd/minikube/cmd/delete.go            |   2 +-
 pkg/drivers/kic/kic.go                |   2 +-
 pkg/drivers/kic/oci/errors.go         |   5 +-
 pkg/drivers/kic/oci/network.go        |  21 +++--
 pkg/drivers/kic/oci/network_create.go | 115 +++++++++++++++++---------
 pkg/drivers/kic/oci/oci.go            |  11 +--
 6 files changed, 101 insertions(+), 55 deletions(-)

diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go
index 9b45c582400d..8ce08f8d8582 100644
--- a/cmd/minikube/cmd/delete.go
+++ b/cmd/minikube/cmd/delete.go
@@ -260,7 +260,7 @@ func deletePossibleKicLeftOver(cname string, driverName string) {
 		klog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs)
 	}
 
-	errs = oci.DeleteKICNetworks()
+	errs = oci.DeleteKICNetworks(bin)
 	if errs != nil {
 		klog.Warningf("error deleting leftover networks (might be okay).\nTo see the list of networks: 'docker network ls'\n:%v", errs)
 	}
diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go
index 4c47a478cf6f..156079675dc1 100644
--- a/pkg/drivers/kic/kic.go
+++ b/pkg/drivers/kic/kic.go
@@ -320,7 +320,7 @@ func (d *Driver) Remove() error {
 		return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id)
 	}
 
-	if err := oci.RemoveNetwork(d.NodeConfig.ClusterName); err != nil {
+	if err := oci.RemoveNetwork(d.OCIBinary, d.NodeConfig.ClusterName); err != nil {
 		klog.Warningf("failed to remove network (which might be okay) %s: %v", d.NodeConfig.ClusterName, err)
 	}
 	return nil
diff --git a/pkg/drivers/kic/oci/errors.go b/pkg/drivers/kic/oci/errors.go
index 6089d88cb7d7..ce4aa00be45d 100644
--- a/pkg/drivers/kic/oci/errors.go
+++ b/pkg/drivers/kic/oci/errors.go
@@ -85,14 +85,15 @@ func LogContainerDebug(ociBin string, name string) string {
 		} else {
 			klog.Infof("postmortem docker info: %+v", di)
 		}
-		logDockerNetworkInspect(name)
+		logDockerNetworkInspect(ociBin, name)
 	} else {
 		pi, err := podmanSystemInfo()
 		if err != nil {
-			klog.Warningf("couldn't get postmortem info, failed to to run podman info: %v", err)
+			klog.Warningf("couldn't get postmortem podman info: %v", err)
 		} else {
 			klog.Infof("postmortem podman info: %+v", pi)
 		}
+		logDockerNetworkInspect(ociBin, name)
 	}
 
 	if rr.Stdout.Len() == 0 {
diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go
index a3c7edb94f6e..de2b742adf2c 100644
--- a/pkg/drivers/kic/oci/network.go
+++ b/pkg/drivers/kic/oci/network.go
@@ -35,7 +35,7 @@ import (
 func RoutableHostIPFromInside(ociBin string, clusterName string, containerName string) (net.IP, error) {
 	if ociBin == Docker {
 		if runtime.GOOS == "linux" {
-			info, err := dockerNetworkInspect(clusterName)
+			info, err := containerNetworkInspect(ociBin, clusterName)
 			if err != nil {
 				if errors.Is(err, ErrNetworkNotFound) {
 					klog.Infof("The container %s is not attached to a network, this could be because the cluster was created by minikube <v1.14, will try to get the IP using container gatway", containerName)
@@ -51,7 +51,7 @@ func RoutableHostIPFromInside(ociBin string, clusterName string, containerName s
 	}
 	// podman
 	if runtime.GOOS == "linux" {
-		return containerGatewayIP(Podman, containerName)
+		return containerGatewayIP(ociBin, containerName)
 	}
 
 	return nil, fmt.Errorf("RoutableHostIPFromInside is currently only implemented for linux")
@@ -128,14 +128,14 @@ func ForwardedPort(ociBin string, ociID string, contPort int) (int, error) {
 // ContainerIPs returns ipv4,ipv6, error of a container by their name
 func ContainerIPs(ociBin string, name string) (string, string, error) {
 	if ociBin == Podman {
-		return podmanContainerIP(name)
+		return podmanContainerIP(ociBin, name)
 	}
-	return dockerContainerIP(name)
+	return dockerContainerIP(ociBin, name)
 }
 
 // podmanContainerIP returns ipv4, ipv6 of container or error
-func podmanContainerIP(name string) (string, string, error) {
-	rr, err := runCmd(exec.Command(Podman, "container", "inspect",
+func podmanContainerIP(ociBin string, name string) (string, string, error) {
+	rr, err := runCmd(exec.Command(ociBin, "container", "inspect",
 		"-f", "{{.NetworkSettings.IPAddress}}",
 		name))
 	if err != nil {
@@ -143,15 +143,20 @@ func podmanContainerIP(name string) (string, string, error) {
 	}
 	output := strings.TrimSpace(rr.Stdout.String())
 	if err == nil && output == "" { // podman returns empty for 127.0.0.1
+		// check network, if the ip address is missing
+		ipv4, ipv6, err := dockerContainerIP(ociBin, name)
+		if err == nil {
+			return ipv4, ipv6, nil
+		}
 		return DefaultBindIPV4, "", nil
 	}
 	return output, "", nil
 }
 
 // dockerContainerIP returns ipv4, ipv6 of container or error
-func dockerContainerIP(name string) (string, string, error) {
+func dockerContainerIP(ociBin string, name string) (string, string, error) {
 	// retrieve the IP address of the node using docker inspect
-	lines, err := inspect(Docker, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}")
+	lines, err := inspect(ociBin, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}")
 	if err != nil {
 		return "", "", errors.Wrap(err, "inspecting NetworkSettings.Networks")
 	}
diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go
index 9b927501f66e..e9421fcb4a27 100644
--- a/pkg/drivers/kic/oci/network_create.go
+++ b/pkg/drivers/kic/oci/network_create.go
@@ -37,20 +37,17 @@ const firstSubnetAddr = "192.168.49.0"
 // big enough for a cluster of 254 nodes
 const defaultSubnetMask = 24
 
-// name of the default Docker bridge network, used to lookup the MTU (see #9528)
-const dockerDefaultBridge = "bridge"
+// name of the default bridge network, used to lookup the MTU (see #9528)
+const defaultBridgeName = "bridge"
 
 // CreateNetwork creates a network returns gateway and error, minikube creates one network per cluster
 func CreateNetwork(ociBin string, name string) (net.IP, error) {
-	if ociBin != Docker {
-		return nil, fmt.Errorf("%s network not implemented yet", ociBin)
-	}
-	return createDockerNetwork(name)
+	return createDockerNetwork(ociBin, name)
 }
 
-func createDockerNetwork(clusterName string) (net.IP, error) {
+func createDockerNetwork(ociBin string, clusterName string) (net.IP, error) {
 	// check if the network already exists
-	info, err := dockerNetworkInspect(clusterName)
+	info, err := containerNetworkInspect(ociBin, clusterName)
 	if err == nil {
 		klog.Infof("Found existing network %+v", info)
 		return info.gateway, nil
@@ -58,16 +55,16 @@ func createDockerNetwork(clusterName string) (net.IP, error) {
 
 	// will try to get MTU from the docker network to avoid issue with systems with exotic MTU settings.
 	// related issue #9528
-	info, err = dockerNetworkInspect(dockerDefaultBridge)
+	info, err = containerNetworkInspect(ociBin, defaultBridgeName)
 	if err != nil {
-		klog.Warningf("failed to get mtu information from the docker's default network %q: %v", dockerDefaultBridge, err)
+		klog.Warningf("failed to get mtu information from the %s's default network %q: %v", ociBin, defaultBridgeName, err)
 	}
 	attempts := 0
 	subnetAddr := firstSubnetAddr
 	// Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work.
 	// will be like 192.168.49.0/24 ,...,192.168.239.0/24
 	for attempts < 20 {
-		info.gateway, err = tryCreateDockerNetwork(subnetAddr, defaultSubnetMask, info.mtu, clusterName)
+		info.gateway, err = tryCreateDockerNetwork(ociBin, subnetAddr, defaultSubnetMask, info.mtu, clusterName)
 		if err == nil {
 			return info.gateway, nil
 		}
@@ -90,7 +87,7 @@ func createDockerNetwork(clusterName string) (net.IP, error) {
 	return info.gateway, fmt.Errorf("failed to create network after 20 attempts")
 }
 
-func tryCreateDockerNetwork(subnetAddr string, subnetMask int, mtu int, name string) (net.IP, error) {
+func tryCreateDockerNetwork(ociBin string, subnetAddr string, subnetMask int, mtu int, name string) (net.IP, error) {
 	gateway := net.ParseIP(subnetAddr)
 	gateway.To4()[3]++ // first ip for gateway
 	klog.Infof("attempt to create network %s/%d with subnet: %s and gateway %s and MTU of %d ...", subnetAddr, subnetMask, name, gateway, mtu)
@@ -100,20 +97,25 @@ func tryCreateDockerNetwork(subnetAddr string, subnetMask int, mtu int, name str
 		"--driver=bridge",
 		fmt.Sprintf("--subnet=%s", fmt.Sprintf("%s/%d", subnetAddr, subnetMask)),
 		fmt.Sprintf("--gateway=%s", gateway),
-		// options documentation https://docs.docker.com/engine/reference/commandline/network_create/#bridge-driver-options
-		"-o", "--ip-masq",
-		"-o", "--icc",
-		fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"),
-		name,
 	}
-
-	// adding MTU option because #9528
-	if mtu > 0 {
+	if ociBin == Docker {
+		// options documentation https://docs.docker.com/engine/reference/commandline/network_create/#bridge-driver-options
+		args = append(args, "-o")
+		args = append(args, "--ip-masq")
 		args = append(args, "-o")
-		args = append(args, fmt.Sprintf("com.docker.network.driver.mtu=%d", mtu))
+		args = append(args, "--icc")
+
+		// adding MTU option because #9528
+		if mtu > 0 {
+			args = append(args, "-o")
+			args = append(args, fmt.Sprintf("com.docker.network.driver.mtu=%d", mtu))
+		}
+
+		args = append(args, fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"))
 	}
+	args = append(args, name)
 
-	rr, err := runCmd(exec.Command(Docker, args...))
+	rr, err := runCmd(exec.Command(ociBin, args...))
 	if err != nil {
 		// Pool overlaps with other one on this address space
 		if strings.Contains(rr.Output(), "Pool overlaps") {
@@ -135,6 +137,16 @@ type netInfo struct {
 	mtu     int
 }
 
+func containerNetworkInspect(ociBin string, name string) (netInfo, error) {
+	if ociBin == Docker {
+		return dockerNetworkInspect(name)
+	}
+	if ociBin == Podman {
+		return podmanNetworkInspect(name)
+	}
+	return netInfo{}, fmt.Errorf("%s unknown", ociBin)
+}
+
 // networkInspect is only used to unmarshal the docker network inspect output and translate it to netInfo
 type networkInspect struct {
 	Name         string
@@ -153,7 +165,7 @@ func dockerNetworkInspect(name string) (netInfo, error) {
 	cmd := exec.Command(Docker, "network", "inspect", name, "--format", `{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{(index .Options "com.docker.network.driver.mtu")}},{{$first := true}} "ContainerIPs": [{{range $k,$v := .Containers }}{{if $first}}{{$first = false}}{{else}}, {{end}}"{{$v.IPv4Address}}"{{end}}]}`)
 	rr, err := runCmd(cmd)
 	if err != nil {
-		logDockerNetworkInspect(name)
+		logDockerNetworkInspect(Docker, name)
 		if strings.Contains(rr.Output(), "No such network") {
 
 			return info, ErrNetworkNotFound
@@ -177,8 +189,39 @@ func dockerNetworkInspect(name string) (netInfo, error) {
 	return info, nil
 }
 
-func logDockerNetworkInspect(name string) {
-	cmd := exec.Command(Docker, "network", "inspect", name)
+func podmanNetworkInspect(name string) (netInfo, error) {
+	var info = netInfo{name: name}
+	cmd := exec.Command(Podman, "network", "inspect", name, "--format", `{{(index .IPAM.Config 0).Subnet}},{{(index .IPAM.Config 0).Gateway}}`)
+	rr, err := runCmd(cmd)
+	if err != nil {
+		logDockerNetworkInspect(Podman, name)
+		if strings.Contains(rr.Output(), "No such network") {
+
+			return info, ErrNetworkNotFound
+		}
+		return info, err
+	}
+
+	// results looks like 172.17.0.0/16,172.17.0.1,1500
+	vals := strings.Split(strings.TrimSpace(rr.Stdout.String()), ",")
+	if len(vals) == 0 {
+		return info, fmt.Errorf("empty list network inspect: %q", rr.Output())
+	}
+
+	if len(vals) > 0 {
+		info.gateway = net.ParseIP(vals[1])
+	}
+
+	_, info.subnet, err = net.ParseCIDR(vals[0])
+	if err != nil {
+		return info, errors.Wrapf(err, "parse subnet for %s", name)
+	}
+
+	return info, nil
+}
+
+func logDockerNetworkInspect(ociBin string, name string) {
+	cmd := exec.Command(ociBin, "network", "inspect", name)
 	klog.Infof("running %v to gather additional debugging logs...", cmd.Args)
 	rr, err := runCmd(cmd)
 	if err != nil {
@@ -188,11 +231,11 @@ func logDockerNetworkInspect(name string) {
 }
 
 // RemoveNetwork removes a network
-func RemoveNetwork(name string) error {
-	if !networkExists(name) {
+func RemoveNetwork(ociBin string, name string) error {
+	if !networkExists(ociBin, name) {
 		return nil
 	}
-	rr, err := runCmd(exec.Command(Docker, "network", "remove", name))
+	rr, err := runCmd(exec.Command(ociBin, "network", "rm", name))
 	if err != nil {
 		if strings.Contains(rr.Output(), "No such network") {
 			return ErrNetworkNotFound
@@ -206,8 +249,8 @@ func RemoveNetwork(name string) error {
 	return err
 }
 
-func networkExists(name string) bool {
-	_, err := dockerNetworkInspect(name)
+func networkExists(ociBin string, name string) bool {
+	_, err := containerNetworkInspect(ociBin, name)
 	if err != nil && !errors.Is(err, ErrNetworkNotFound) { // log unexpected error
 		klog.Warningf("Error inspecting docker network %s: %v", name, err)
 	}
@@ -216,12 +259,8 @@ func networkExists(name string) bool {
 
 // networkNamesByLabel returns all network names created by a label
 func networkNamesByLabel(ociBin string, label string) ([]string, error) {
-	if ociBin != Docker {
-		return nil, fmt.Errorf("%s not supported", ociBin)
-	}
-
 	// docker network ls --filter='label=created_by.minikube.sigs.k8s.io=true' --format '{{.Name}}'
-	rr, err := runCmd(exec.Command(Docker, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}"))
+	rr, err := runCmd(exec.Command(ociBin, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}"))
 	if err != nil {
 		return nil, err
 	}
@@ -235,14 +274,14 @@ func networkNamesByLabel(ociBin string, label string) ([]string, error) {
 }
 
 // DeleteKICNetworks deletes all networks created by kic
-func DeleteKICNetworks() []error {
+func DeleteKICNetworks(ociBin string) []error {
 	var errs []error
-	ns, err := networkNamesByLabel(Docker, CreatedByLabelKey+"=true")
+	ns, err := networkNamesByLabel(ociBin, CreatedByLabelKey)
 	if err != nil {
 		return []error{errors.Wrap(err, "list all volume")}
 	}
 	for _, n := range ns {
-		err := RemoveNetwork(n)
+		err := RemoveNetwork(ociBin, n)
 		if err != nil {
 			errs = append(errs, err)
 		}
diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go
index 087d6224f4b6..63c45fc2833d 100644
--- a/pkg/drivers/kic/oci/oci.go
+++ b/pkg/drivers/kic/oci/oci.go
@@ -146,6 +146,12 @@ func CreateContainerNode(p CreateParams) error {
 		// label th enode wuth the node ID
 		"--label", p.NodeLabel,
 	}
+	// to provide a static IP
+	if p.Network != "" && p.IP != "" {
+		runArgs = append(runArgs, "--network", p.Network)
+		runArgs = append(runArgs, "--ip", p.IP)
+	}
+
 	memcgSwap := true
 	if runtime.GOOS == "linux" {
 		if _, err := os.Stat("/sys/fs/cgroup/memory/memsw.limit_in_bytes"); os.IsNotExist(err) {
@@ -170,11 +176,6 @@ func CreateContainerNode(p CreateParams) error {
 		virtualization = "podman" // VIRTUALIZATION_PODMAN
 	}
 	if p.OCIBinary == Docker {
-		// to provide a static IP for docker
-		if p.Network != "" && p.IP != "" {
-			runArgs = append(runArgs, "--network", p.Network)
-			runArgs = append(runArgs, "--ip", p.IP)
-		}
 		runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name))
 		// ignore apparmore github actions docker: https://github.com/kubernetes/minikube/issues/7624
 		runArgs = append(runArgs, "--security-opt", "apparmor=unconfined")

From d370a3e164485efcf295fcde723e2fe2ab941f31 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 23 Nov 2020 12:56:48 -0800
Subject: [PATCH 04/45] fix up script

---
 hack/metrics/metrics.go                | 11 +++++------
 site/content/en/docs/commands/start.md |  1 +
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go
index 8a7582cada9a..ad8602beb20a 100644
--- a/hack/metrics/metrics.go
+++ b/hack/metrics/metrics.go
@@ -42,7 +42,7 @@ const (
 )
 
 var (
-	// The task latency in milliseconds.
+	// The task latency in seconds
 	latencyS = stats.Float64("repl/start_time", "start time in seconds", "s")
 )
 
@@ -56,7 +56,7 @@ func main() {
 func execute() error {
 	projectID := os.Getenv(projectEnvVar)
 	if projectID == "" {
-		return nil, fmt.Errorf("GCP tracer requires a valid GCP project id set via the %s env variable", projectEnvVar)
+		return fmt.Errorf("metrics collector requires a valid GCP project id set via the %s env variable", projectEnvVar)
 	}
 
 	osMethod, err := tag.NewKey("os")
@@ -64,7 +64,7 @@ func execute() error {
 		return errors.Wrap(err, "new tag key")
 	}
 
-	ctx, err := tag.New(context.Background(), tag.Insert(osMethod, runtime.GOOS), tag.Insert(driverMethod, "docker"))
+	ctx, err := tag.New(context.Background(), tag.Insert(osMethod, runtime.GOOS))
 	if err != nil {
 		return errors.Wrap(err, "new tag")
 	}
@@ -102,17 +102,16 @@ func execute() error {
 	}
 	defer sd.StopMetricsExporter()
 
-	// Record 2p0 start time values
 	for {
 		st := minikubeStartTime(ctx)
 		fmt.Printf("Latency: %f\n", st)
 		stats.Record(ctx, latencyS.M(st))
-		time.Sleep(2 * time.Second)
+		time.Sleep(30 * time.Second)
 	}
 }
 
 func minikubeStartTime(ctx context.Context) (float64, error) {
-	defer deleteMinikube(ctx, profile)
+	defer deleteMinikube(ctx)
 
 	minikube := filepath.Join(os.Getenv("HOME"), "minikube/out/minikube")
 
diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index 122f0b407e80..2cc6dd0d8bee 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -87,6 +87,7 @@ minikube start [flags]
       --preload                           If set, download tarball of preloaded images if available to improve start time. Defaults to true. (default true)
       --registry-mirror strings           Registry mirrors to pass to the Docker daemon
       --service-cluster-ip-range string   The CIDR to be used for service cluster IPs. (default "10.96.0.0/12")
+      --trace string                      Send trace events. Options include: [gcp]
       --uuid string                       Provide VM UUID to restore MAC address (hyperkit driver only)
       --vm                                Filter to use only VM Drivers
       --vm-driver driver                  DEPRECATED, use driver instead.

From 46f82272e01a33eb029d7274dcc51a34558d2642 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 23 Nov 2020 16:14:06 -0800
Subject: [PATCH 05/45] Add metrics script to collect data for cloud monitoring

---
 go.sum                   |  3 +++
 hack/metrics/metrics.go  | 28 ++++++++++++--------
 hack/metrics/minikube.go | 57 +++++++++++++++++++++++++++++++++++++---
 3 files changed, 74 insertions(+), 14 deletions(-)

diff --git a/go.sum b/go.sum
index 22abe6c76e85..d1f38e98722b 100644
--- a/go.sum
+++ b/go.sum
@@ -40,6 +40,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 code.gitea.io/sdk/gitea v0.12.0/go.mod h1:z3uwDV/b9Ls47NGukYM9XhnHtqPh/J+t40lsUrR6JDY=
 contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA=
 contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0=
+contrib.go.opencensus.io/exporter/stackdriver v0.12.1 h1:Dll2uFfOVI3fa8UzsHyP6z0M6fEc9ZTAMo+Y3z282Xg=
 contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw=
 contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE=
 contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA=
@@ -137,6 +138,7 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ=
+github.com/apex/log v1.3.0 h1:1fyfbPvUwD10nMoh3hY6MXzvZShJQn9/ck7ATgAt5pA=
 github.com/apex/log v1.3.0/go.mod h1:jd8Vpsr46WAe3EZSQ/IUMs2qQD/GOycT5rPWCO1yGcs=
 github.com/apex/logs v0.0.4/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
 github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
@@ -202,6 +204,7 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH
 github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc=
 github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6ZsFhtM8HrDku0pxJ3/Lr51rwykrzgFwpmTzleatY=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go
index ad8602beb20a..04b26fbed55d 100644
--- a/hack/metrics/metrics.go
+++ b/hack/metrics/metrics.go
@@ -22,7 +22,6 @@ import (
 	"log"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"runtime"
 	"time"
 
@@ -98,12 +97,16 @@ func execute() error {
 	trace.RegisterExporter(sd)
 
 	if err := sd.StartMetricsExporter(); err != nil {
-		log.Fatalf("Error starting metric exporter: %v", err)
+		return errors.Wrap(err, "starting metric exporter")
 	}
 	defer sd.StopMetricsExporter()
 
 	for {
-		st := minikubeStartTime(ctx)
+		st, err := minikubeStartTime(ctx)
+		if err != nil {
+			log.Printf("error collecting start time: %v", err)
+			continue
+		}
 		fmt.Printf("Latency: %f\n", st)
 		stats.Record(ctx, latencyS.M(st))
 		time.Sleep(30 * time.Second)
@@ -111,24 +114,27 @@ func execute() error {
 }
 
 func minikubeStartTime(ctx context.Context) (float64, error) {
-	defer deleteMinikube(ctx)
-
-	minikube := filepath.Join(os.Getenv("HOME"), "minikube/out/minikube")
+	minikubePath, err := downloadMinikube()
+	if err != nil {
+		return 0, errors.Wrap(err, "downloading minikube")
+	}
+	defer os.Remove(minikubePath)
+	defer deleteMinikube(ctx, minikubePath)
 
-	cmd := exec.CommandContext(ctx, minikube, "start", "--driver=docker", "-p", profile)
+	cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile)
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
 
 	t := time.Now()
 	log.Print("Running minikube start....")
 	if err := cmd.Run(); err != nil {
-		log.Fatal(err)
+		return 0, errors.Wrapf(err, "running %v", cmd.Args)
 	}
-	return time.Since(t).Seconds()
+	return time.Since(t).Seconds(), nil
 }
 
-func deleteMinikube(ctx context.Context) {
-	cmd := exec.CommandContext(ctx, minikube, "delete", "-p", profile)
+func deleteMinikube(ctx context.Context, minikubePath string) {
+	cmd := exec.CommandContext(ctx, minikubePath, "delete", "-p", profile)
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {
diff --git a/hack/metrics/minikube.go b/hack/metrics/minikube.go
index 712a8dd51636..d6b7adf86c0c 100644
--- a/hack/metrics/minikube.go
+++ b/hack/metrics/minikube.go
@@ -16,11 +16,62 @@ limitations under the License.
 
 package main
 
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"runtime"
+
+	"log"
+
+	"cloud.google.com/go/storage"
+	"github.com/pkg/errors"
+)
+
 const (
-	repo = "https://github.com/kubernetes/minikube"
+	bucketName = "priya-test-bucket/latest"
 )
 
-// buildMinikubeAtHEAD copies minikube to a temp dir
-func buildMinikubeAtHEAD() error {
+// download minikube latest to a tmp file
+func downloadMinikube() (string, error) {
+	b := binary()
+	tmp, err := ioutil.TempFile("", b)
+	if err != nil {
+		return "", errors.Wrap(err, "creating tmp file")
+	}
+	if err := tmp.Close(); err != nil {
+		return "", errors.Wrap(err, "closing tmp file")
+	}
+	client, err := storage.NewClient(context.Background())
+	if err != nil {
+		return "", errors.Wrap(err, "creating client")
+	}
+	ctx := context.Background()
+	rc, err := client.Bucket(bucketName).Object(b).NewReader(ctx)
+	if err != nil {
+		return "", errors.Wrap(err, "gcs new reader")
+	}
+	defer rc.Close()
+
+	data, err := ioutil.ReadAll(rc)
+	if err != nil {
+		return "", errors.Wrap(err, "ioutil read all")
+	}
+	log.Printf("downloading gs://%s/%s to %v", bucketName, b, tmp.Name())
+	if err := ioutil.WriteFile(tmp.Name(), data, 0777); err != nil {
+		return "", errors.Wrap(err, "writing file")
+	}
+	if err := os.Chmod(tmp.Name(), 0700); err != nil {
+		return "", errors.Wrap(err, "chmod")
+	}
+	return tmp.Name(), nil
+}
 
+func binary() string {
+	b := fmt.Sprintf("minikube-%s-amd64", runtime.GOOS)
+	if runtime.GOOS == "windows" {
+		b += ".exe"
+	}
+	return b
 }

From b01bad0ebf63c820c1b5e78874ce7056a009c849 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 23 Nov 2020 16:18:46 -0800
Subject: [PATCH 06/45] add --memory flag to constrain that variable

---
 hack/metrics/metrics.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go
index 04b26fbed55d..0c0b7153b58d 100644
--- a/hack/metrics/metrics.go
+++ b/hack/metrics/metrics.go
@@ -121,7 +121,7 @@ func minikubeStartTime(ctx context.Context) (float64, error) {
 	defer os.Remove(minikubePath)
 	defer deleteMinikube(ctx, minikubePath)
 
-	cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile)
+	cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile, "--memory=2000")
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
 

From 5b2a552091a905225d84c8f0eaf0bf298a70738d Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 23 Nov 2020 16:21:13 -0800
Subject: [PATCH 07/45] cleanup

---
 hack/metrics/README.md  | 2 +-
 hack/metrics/metrics.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/hack/metrics/README.md b/hack/metrics/README.md
index ab82671fc6f2..68df5ac8387b 100644
--- a/hack/metrics/README.md
+++ b/hack/metrics/README.md
@@ -1,4 +1,4 @@
-This script runs `minikube start` in a loop times how long it takes.
+This script runs `minikube start` in a loop and measures how long it takes.
 It exports this data to Stackdriver via the OpenTelemetry API.
 
 This script is used to track minikube performance and prevent regressions.
diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go
index 0c0b7153b58d..bf74e3ba3984 100644
--- a/hack/metrics/metrics.go
+++ b/hack/metrics/metrics.go
@@ -76,7 +76,7 @@ func execute() error {
 		Aggregation: view.LastValue(),
 	}
 	if err := view.Register(v); err != nil {
-		log.Fatalf("Failed to register the view: %v", err)
+		return errors.Wrap(err, "registering view")
 	}
 
 	sd, err := stackdriver.NewExporter(stackdriver.Options{

From 2a0a01a18cd93046ae2c0b70045783d5754eae9c Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 24 Nov 2020 10:14:09 -0800
Subject: [PATCH 08/45] UPDate

---
 hack/metrics/metrics.go | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go
index bf74e3ba3984..142d3e201331 100644
--- a/hack/metrics/metrics.go
+++ b/hack/metrics/metrics.go
@@ -22,7 +22,6 @@ import (
 	"log"
 	"os"
 	"os/exec"
-	"runtime"
 	"time"
 
 	_ "cloud.google.com/go/storage"
@@ -63,10 +62,6 @@ func execute() error {
 		return errors.Wrap(err, "new tag key")
 	}
 
-	ctx, err := tag.New(context.Background(), tag.Insert(osMethod, runtime.GOOS))
-	if err != nil {
-		return errors.Wrap(err, "new tag")
-	}
 	// Register the view. It is imperative that this step exists,
 	// otherwise recorded metrics will be dropped and never exported.
 	v := &view.View{
@@ -74,6 +69,7 @@ func execute() error {
 		Measure:     latencyS,
 		Description: "minikube start time",
 		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{osMethod},
 	}
 	if err := view.Register(v); err != nil {
 		return errors.Wrap(err, "registering view")
@@ -100,7 +96,7 @@ func execute() error {
 		return errors.Wrap(err, "starting metric exporter")
 	}
 	defer sd.StopMetricsExporter()
-
+	ctx := context.Background()
 	for {
 		st, err := minikubeStartTime(ctx)
 		if err != nil {

From ad3e4a98d0e513eb8a9a6d9319e30125c41bdb10 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 24 Nov 2020 11:36:34 -0800
Subject: [PATCH 09/45] Add --cancel-scheduled-stop flag and add to integration
 test to make sure it works

---
 cmd/minikube/cmd/stop.go                |  8 ++++++
 test/integration/scheduled_stop_test.go | 37 ++++++++++++++++---------
 2 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/cmd/minikube/cmd/stop.go b/cmd/minikube/cmd/stop.go
index 4cce5d09fa86..e8d537e9c5ce 100644
--- a/cmd/minikube/cmd/stop.go
+++ b/cmd/minikube/cmd/stop.go
@@ -46,6 +46,7 @@ var (
 	stopAll               bool
 	keepActive            bool
 	scheduledStopDuration time.Duration
+	cancelScheduledStop   bool
 )
 
 // stopCmd represents the stop command
@@ -60,6 +61,8 @@ func init() {
 	stopCmd.Flags().BoolVar(&stopAll, "all", false, "Set flag to stop all profiles (clusters)")
 	stopCmd.Flags().BoolVar(&keepActive, "keep-context-active", false, "keep the kube-context active after cluster is stopped. Defaults to false.")
 	stopCmd.Flags().DurationVar(&scheduledStopDuration, "schedule", 0*time.Second, "Set flag to stop cluster after a set amount of time (e.g. --schedule=5m)")
+	stopCmd.Flags().BoolVar(&cancelScheduledStop, "cancel-scheduled-stop", false, "cancel any existing scheduled stop requests")
+
 	if err := stopCmd.Flags().MarkHidden("schedule"); err != nil {
 		klog.Info("unable to mark --schedule flag as hidden")
 	}
@@ -97,6 +100,11 @@ func runStop(cmd *cobra.Command, args []string) {
 
 	// Kill any existing scheduled stops
 	schedule.KillExisting(profilesToStop)
+	if cancelScheduledStop {
+		register.Reg.SetStep(register.Done)
+		out.Step(style.Stopped, `All existing scheduled stops cancelled`)
+		return
+	}
 
 	if scheduledStopDuration != 0 {
 		if err := schedule.Daemonize(profilesToStop, scheduledStopDuration); err != nil {
diff --git a/test/integration/scheduled_stop_test.go b/test/integration/scheduled_stop_test.go
index a9d833a22ba4..597f9a2200bf 100644
--- a/test/integration/scheduled_stop_test.go
+++ b/test/integration/scheduled_stop_test.go
@@ -50,7 +50,7 @@ func TestScheduledStopWindows(t *testing.T) {
 	startMinikube(ctx, t, profile)
 
 	// schedule a stop for 5m from now
-	scheduledStopMinikube(ctx, t, profile, "5m")
+	stopMinikube(ctx, t, profile, []string{"--schedule", "5m"})
 
 	// make sure the systemd service is running
 	rr, err := Run(t, exec.CommandContext(ctx, Target(), []string{"ssh", "-p", profile, "--", "sudo", "systemctl", "show", constants.ScheduledStopSystemdService, "--no-page"}...))
@@ -62,12 +62,12 @@ func TestScheduledStopWindows(t *testing.T) {
 	}
 
 	// reschedule stop for 5 seconds from now
-	scheduledStopMinikube(ctx, t, profile, "5s")
+	stopMinikube(ctx, t, profile, []string{"--schedule", "5s"})
 
 	// sleep for 5 seconds
 	time.Sleep(5 * time.Second)
 	// make sure minikube status is "Stopped"
-	ensureMinikubeStatusStopped(ctx, t, profile)
+	ensureMinikubeStatus(ctx, t, profile, state.Stopped.String())
 }
 
 func TestScheduledStopUnix(t *testing.T) {
@@ -83,20 +83,31 @@ func TestScheduledStopUnix(t *testing.T) {
 	startMinikube(ctx, t, profile)
 
 	// schedule a stop for 5 min from now and make sure PID is created
-	scheduledStopMinikube(ctx, t, profile, "5m")
+	stopMinikube(ctx, t, profile, []string{"--schedule", "5m"})
 	pid := checkPID(t, profile)
 	if !processRunning(t, pid) {
 		t.Fatalf("process %v is not running", pid)
 	}
 
-	// redo scheduled stop to be in 10s
-	scheduledStopMinikube(ctx, t, profile, "10s")
+	// schedule a second stop which should cancel the first scheduled stop
+	stopMinikube(ctx, t, profile, []string{"--schedule", "8s"})
 	if processRunning(t, pid) {
 		t.Fatalf("process %v running but should have been killed on reschedule of stop", pid)
 	}
 	checkPID(t, profile)
-	// make sure minikube status is "Stopped"
-	ensureMinikubeStatusStopped(ctx, t, profile)
+
+	// cancel the shutdown and make sure minikube is still running after 8 seconds
+	// sleep 12 just to be safe
+	stopMinikube(ctx, t, profile, []string{"--cancel-scheduled-stop"})
+	time.Sleep(12 * time.Second)
+	ensureMinikubeStatus(ctx, t, profile, state.Running.String())
+
+	// schedule another stop, make sure minikube status is "Stopped"
+	stopMinikube(ctx, t, profile, []string{"--schedule", "5s"})
+	if processRunning(t, pid) {
+		t.Fatalf("process %v running but should have been killed on reschedule of stop", pid)
+	}
+	ensureMinikubeStatus(ctx, t, profile, state.Stopped.String())
 }
 
 func startMinikube(ctx context.Context, t *testing.T, profile string) {
@@ -107,8 +118,9 @@ func startMinikube(ctx context.Context, t *testing.T, profile string) {
 	}
 }
 
-func scheduledStopMinikube(ctx context.Context, t *testing.T, profile string, stop string) {
-	args := []string{"stop", "-p", profile, "--schedule", stop}
+func stopMinikube(ctx context.Context, t *testing.T, profile string, additionalArgs []string) {
+	args := []string{"stop", "-p", profile}
+	args = append(args, additionalArgs...)
 	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
 	if err != nil {
 		t.Fatalf("starting minikube: %v\n%s", err, rr.Output())
@@ -144,14 +156,13 @@ func processRunning(t *testing.T, pid string) bool {
 	t.Log("signal error was: ", err)
 	return err == nil
 }
-
-func ensureMinikubeStatusStopped(ctx context.Context, t *testing.T, profile string) {
+func ensureMinikubeStatus(ctx context.Context, t *testing.T, profile, wantStatus string) {
 	// wait allotted time to make sure minikube status is "Stopped"
 	checkStatus := func() error {
 		ctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Second))
 		defer cancel()
 		got := Status(ctx, t, Target(), profile, "Host", profile)
-		if got != state.Stopped.String() {
+		if got != wantStatus {
 			return fmt.Errorf("expected post-stop host status to be -%q- but got *%q*", state.Stopped, got)
 		}
 		return nil

From b7eaaf8c7512c66b593c475f76a64d068b3e04f1 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 24 Nov 2020 11:36:41 -0800
Subject: [PATCH 10/45] update docs

---
 site/content/en/docs/commands/stop.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/site/content/en/docs/commands/stop.md b/site/content/en/docs/commands/stop.md
index 9f54204ab185..d5a2c19563f2 100644
--- a/site/content/en/docs/commands/stop.md
+++ b/site/content/en/docs/commands/stop.md
@@ -20,9 +20,10 @@ minikube stop [flags]
 ### Options
 
 ```
-      --all                   Set flag to stop all profiles (clusters)
-      --keep-context-active   keep the kube-context active after cluster is stopped. Defaults to false.
-  -o, --output string         Format to print stdout in. Options include: [text,json] (default "text")
+      --all                     Set flag to stop all profiles (clusters)
+      --cancel-scheduled-stop   cancel any existing scheduled stop requests
+      --keep-context-active     keep the kube-context active after cluster is stopped. Defaults to false.
+  -o, --output string           Format to print stdout in. Options include: [text,json] (default "text")
 ```
 
 ### Options inherited from parent commands

From 70cb886a393d1ebd55efa5cf470fbce944e5d0a2 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 24 Nov 2020 17:29:46 -0800
Subject: [PATCH 11/45] shorten flag name to --cancel-scheduled

---
 cmd/minikube/cmd/stop.go                | 2 +-
 site/content/en/docs/commands/stop.md   | 8 ++++----
 test/integration/scheduled_stop_test.go | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/cmd/minikube/cmd/stop.go b/cmd/minikube/cmd/stop.go
index e8d537e9c5ce..4fd0fadabc49 100644
--- a/cmd/minikube/cmd/stop.go
+++ b/cmd/minikube/cmd/stop.go
@@ -61,7 +61,7 @@ func init() {
 	stopCmd.Flags().BoolVar(&stopAll, "all", false, "Set flag to stop all profiles (clusters)")
 	stopCmd.Flags().BoolVar(&keepActive, "keep-context-active", false, "keep the kube-context active after cluster is stopped. Defaults to false.")
 	stopCmd.Flags().DurationVar(&scheduledStopDuration, "schedule", 0*time.Second, "Set flag to stop cluster after a set amount of time (e.g. --schedule=5m)")
-	stopCmd.Flags().BoolVar(&cancelScheduledStop, "cancel-scheduled-stop", false, "cancel any existing scheduled stop requests")
+	stopCmd.Flags().BoolVar(&cancelScheduledStop, "cancel-scheduled", false, "cancel any existing scheduled stop requests")
 
 	if err := stopCmd.Flags().MarkHidden("schedule"); err != nil {
 		klog.Info("unable to mark --schedule flag as hidden")
diff --git a/site/content/en/docs/commands/stop.md b/site/content/en/docs/commands/stop.md
index d5a2c19563f2..8b96e3de894c 100644
--- a/site/content/en/docs/commands/stop.md
+++ b/site/content/en/docs/commands/stop.md
@@ -20,10 +20,10 @@ minikube stop [flags]
 ### Options
 
 ```
-      --all                     Set flag to stop all profiles (clusters)
-      --cancel-scheduled-stop   cancel any existing scheduled stop requests
-      --keep-context-active     keep the kube-context active after cluster is stopped. Defaults to false.
-  -o, --output string           Format to print stdout in. Options include: [text,json] (default "text")
+      --all                   Set flag to stop all profiles (clusters)
+      --cancel-scheduled      cancel any existing scheduled stop requests
+      --keep-context-active   keep the kube-context active after cluster is stopped. Defaults to false.
+  -o, --output string         Format to print stdout in. Options include: [text,json] (default "text")
 ```
 
 ### Options inherited from parent commands
diff --git a/test/integration/scheduled_stop_test.go b/test/integration/scheduled_stop_test.go
index 597f9a2200bf..508a38599969 100644
--- a/test/integration/scheduled_stop_test.go
+++ b/test/integration/scheduled_stop_test.go
@@ -98,7 +98,7 @@ func TestScheduledStopUnix(t *testing.T) {
 
 	// cancel the shutdown and make sure minikube is still running after 8 seconds
 	// sleep 12 just to be safe
-	stopMinikube(ctx, t, profile, []string{"--cancel-scheduled-stop"})
+	stopMinikube(ctx, t, profile, []string{"--cancel-scheduled"})
 	time.Sleep(12 * time.Second)
 	ensureMinikubeStatus(ctx, t, profile, state.Running.String())
 

From fc8fd7bf273b27fdc197c1bffd65a1fb72d54a90 Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 25 Nov 2020 15:17:54 -0800
Subject: [PATCH 12/45] Update conformance tests for multinode testing:

---
 hack/conformance_tests.sh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/hack/conformance_tests.sh b/hack/conformance_tests.sh
index b4509e8e9cbd..cec198a74148 100755
--- a/hack/conformance_tests.sh
+++ b/hack/conformance_tests.sh
@@ -27,19 +27,21 @@ set -ex -o pipefail
 readonly PROFILE_NAME="k8sconformance"
 readonly MINIKUBE=${1:-./out/minikube}
 shift || true
+readonly START_ARGS=${@:-}
 
 # Requires a fully running Kubernetes cluster.
 "${MINIKUBE}" delete -p "${PROFILE_NAME}" || true
-"${MINIKUBE}" start -p "${PROFILE_NAME}" --wait=all
+"${MINIKUBE}" start -p "${PROFILE_NAME}" ${START_ARGS} --wait=all --nodes=2
 kubectl --context "${PROFILE_NAME}" get pods --all-namespaces
 "${MINIKUBE}" status -p "${PROFILE_NAME}"
 
-go get -u -v github.com/vmware-tanzu/sonobuoy
+curl -LO https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.19.0/sonobuoy_0.19.0_linux_amd64.tar.gz || true
+tar -xzf sonobuoy_0.19.0_linux_amd64.tar.gz
 
 
-sonobuoy run --wait
+./sonobuoy run --mode=certified-conformance --wait --alsologtostderr
 outdir="$(mktemp -d)"
-sonobuoy retrieve "${outdir}"
+./sonobuoy retrieve "${outdir}"
 
 cwd=$(pwd)
 

From 22a99c55cd73cb26cada21dd2e272b24a1b8ac0e Mon Sep 17 00:00:00 2001
From: Sharif Elgamal <selgamal@google.com>
Date: Wed, 25 Nov 2020 17:48:37 -0800
Subject: [PATCH 13/45] small fixes

---
 hack/conformance_tests.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hack/conformance_tests.sh b/hack/conformance_tests.sh
index cec198a74148..439649449f98 100755
--- a/hack/conformance_tests.sh
+++ b/hack/conformance_tests.sh
@@ -48,9 +48,9 @@ cwd=$(pwd)
 cd "${outdir}"
 mkdir ./results; tar xzf *.tar.gz -C ./results
 
-version=$(${MINIKUBE} version  | cut -d" " -f3)
+version=$(${cwd}/${MINIKUBE} version  | cut -d" " -f3)
 
-mkdir "minikube-${version}"
+mkdir -p "minikube-${version}"
 cd "minikube-${version}"
 
 cat <<EOF >PRODUCT.yaml
@@ -69,6 +69,6 @@ cat <<EOF >README.md
 ./hack/conformance_tests.sh $MINIKUBE $START_ARGS
 EOF
 
-cp ../results/plugins/e2e/results/* .
+cp -r ../results/plugins/e2e/results/global/* .
 cd ..
 cp -r "minikube-${version}" "${cwd}"

From 6b4dc381f98f97ef99489abd2ee7c3e59c4eb714 Mon Sep 17 00:00:00 2001
From: Brian Li <brian14708@gmail.com>
Date: Fri, 27 Nov 2020 10:14:01 +0800
Subject: [PATCH 14/45] fix base image when using with custom image repository

---
 pkg/minikube/node/cache.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go
index 3017d31472ce..e775ed3ca104 100644
--- a/pkg/minikube/node/cache.go
+++ b/pkg/minikube/node/cache.go
@@ -124,6 +124,7 @@ func beginDownloadKicBaseImage(g *errgroup.Group, cc *config.ClusterConfig, down
 		baseImg := cc.KicBaseImage
 		if baseImg == kic.BaseImage && len(cc.KubernetesConfig.ImageRepository) != 0 {
 			baseImg = strings.Replace(baseImg, "gcr.io/k8s-minikube", cc.KubernetesConfig.ImageRepository, 1)
+			cc.KicBaseImage = baseImg
 		}
 		var finalImg string
 		// If we end up using a fallback image, notify the user

From a2bc2fe64cafcae02a6a24f87ae0bbc6d36afd18 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 30 Nov 2020 10:24:15 -0800
Subject: [PATCH 15/45] Enable tracing when collecting metrics

---
 hack/metrics/README.md   |  8 ++++++++
 hack/metrics/metrics.go  | 13 +++++++------
 hack/metrics/minikube.go |  2 +-
 pkg/trace/gcp.go         |  7 ++++---
 4 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/hack/metrics/README.md b/hack/metrics/README.md
index 68df5ac8387b..03dd6187bd6a 100644
--- a/hack/metrics/README.md
+++ b/hack/metrics/README.md
@@ -1,4 +1,12 @@
 This script runs `minikube start` in a loop and measures how long it takes.
 It exports this data to Stackdriver via the OpenTelemetry API.
 
+To run this script, run:
+
+```
+MINIKUBE_GCP_PROJECT_ID=<GCP Project ID> go run hack/metrics/*.go
+```
+
 This script is used to track minikube performance and prevent regressions.
+
+_Note: this script will export data to both Cloud Monitoring and Cloud Trace in the provided GCP project_
diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go
index 142d3e201331..363511b6a8b3 100644
--- a/hack/metrics/metrics.go
+++ b/hack/metrics/metrics.go
@@ -31,10 +31,10 @@ import (
 	"go.opencensus.io/stats/view"
 	"go.opencensus.io/tag"
 	"go.opencensus.io/trace"
+	pkgtrace "k8s.io/minikube/pkg/trace"
 )
 
 const (
-	projectEnvVar    = "MINIKUBE_GCP_PROJECT_ID"
 	customMetricName = "custom.googleapis.com/minikube/start_time"
 	profile          = "cloud-monitoring"
 )
@@ -52,9 +52,9 @@ func main() {
 }
 
 func execute() error {
-	projectID := os.Getenv(projectEnvVar)
+	projectID := os.Getenv(pkgtrace.ProjectEnvVar)
 	if projectID == "" {
-		return fmt.Errorf("metrics collector requires a valid GCP project id set via the %s env variable", projectEnvVar)
+		return fmt.Errorf("metrics collector requires a valid GCP project id set via the %s env variable", pkgtrace.ProjectEnvVar)
 	}
 
 	osMethod, err := tag.NewKey("os")
@@ -98,7 +98,7 @@ func execute() error {
 	defer sd.StopMetricsExporter()
 	ctx := context.Background()
 	for {
-		st, err := minikubeStartTime(ctx)
+		st, err := minikubeStartTime(ctx, projectID)
 		if err != nil {
 			log.Printf("error collecting start time: %v", err)
 			continue
@@ -109,7 +109,7 @@ func execute() error {
 	}
 }
 
-func minikubeStartTime(ctx context.Context) (float64, error) {
+func minikubeStartTime(ctx context.Context, projectID string) (float64, error) {
 	minikubePath, err := downloadMinikube()
 	if err != nil {
 		return 0, errors.Wrap(err, "downloading minikube")
@@ -117,7 +117,8 @@ func minikubeStartTime(ctx context.Context) (float64, error) {
 	defer os.Remove(minikubePath)
 	defer deleteMinikube(ctx, minikubePath)
 
-	cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile, "--memory=2000")
+	cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile, "--memory=2000", "--trace=gcp")
+	cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%s", pkgtrace.ProjectEnvVar, projectID))
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
 
diff --git a/hack/metrics/minikube.go b/hack/metrics/minikube.go
index d6b7adf86c0c..16086873e58d 100644
--- a/hack/metrics/minikube.go
+++ b/hack/metrics/minikube.go
@@ -30,7 +30,7 @@ import (
 )
 
 const (
-	bucketName = "priya-test-bucket/latest"
+	bucketName = "minikube/latest"
 )
 
 // download minikube latest to a tmp file
diff --git a/pkg/trace/gcp.go b/pkg/trace/gcp.go
index 8a4bf57b0da7..212d198fb582 100644
--- a/pkg/trace/gcp.go
+++ b/pkg/trace/gcp.go
@@ -31,7 +31,8 @@ import (
 )
 
 const (
-	projectEnvVar = "MINIKUBE_GCP_PROJECT_ID"
+	// ProjectEnvVar is the name of the env variable that the user must pass in their GCP project ID through
+	ProjectEnvVar = "MINIKUBE_GCP_PROJECT_ID"
 	// this is the name of the parent span to help identify it
 	// in the Cloud Trace UI.
 	parentSpanName = "minikube start"
@@ -72,9 +73,9 @@ func (t *gcpTracer) Cleanup() {
 }
 
 func initGCPTracer() (*gcpTracer, error) {
-	projectID := os.Getenv(projectEnvVar)
+	projectID := os.Getenv(ProjectEnvVar)
 	if projectID == "" {
-		return nil, fmt.Errorf("GCP tracer requires a valid GCP project id set via the %s env variable", projectEnvVar)
+		return nil, fmt.Errorf("GCP tracer requires a valid GCP project id set via the %s env variable", ProjectEnvVar)
 	}
 
 	_, flush, err := texporter.InstallNewPipeline(

From 2b43ff99899ce1fdbf47c7280276b5d15d1db929 Mon Sep 17 00:00:00 2001
From: Steven Powell <spowellgrr@gmail.com>
Date: Mon, 30 Nov 2020 15:46:05 -0700
Subject: [PATCH 16/45] Auto detect user OS and default tabs to it

---
 site/content/en/docs/drivers/vmware.md   | 12 ++++----
 site/content/en/docs/handbook/pushing.md | 12 ++++----
 site/content/en/docs/start/_index.md     | 12 ++++----
 site/layouts/shortcodes/linuxtab.html    |  3 ++
 site/layouts/shortcodes/mactab.html      |  3 ++
 site/layouts/shortcodes/windowstab.html  |  3 ++
 site/static/js/tabs.js                   | 36 ++++++++++++++++++------
 7 files changed, 54 insertions(+), 27 deletions(-)
 create mode 100644 site/layouts/shortcodes/linuxtab.html
 create mode 100644 site/layouts/shortcodes/mactab.html
 create mode 100644 site/layouts/shortcodes/windowstab.html

diff --git a/site/content/en/docs/drivers/vmware.md b/site/content/en/docs/drivers/vmware.md
index 7351904e4e1b..81b2ba301069 100644
--- a/site/content/en/docs/drivers/vmware.md
+++ b/site/content/en/docs/drivers/vmware.md
@@ -10,15 +10,15 @@ aliases:
 The vmware driver supports virtualization across all VMware based hypervisors.
 
 {{% tabs %}}
-{{% tab "macOS" %}}
+{{% mactab %}}
 {{% readfile file="/docs/drivers/includes/vmware_macos_usage.inc" %}}
-{{% /tab %}}
-{{% tab "Linux" %}}
+{{% /mactab %}}
+{{% linuxtab %}}
 No documentation is available yet.
-{{% /tab %}}
-{{% tab "Windows" %}}
+{{% /linuxtab %}}
+{{% windowstab %}}
 No documentation is available yet.
-{{% /tab %}}
+{{% /windowstab %}}
 {{% /tabs %}}
 
 ## Issues
diff --git a/site/content/en/docs/handbook/pushing.md b/site/content/en/docs/handbook/pushing.md
index a7419a58b22d..5a43517bdc93 100644
--- a/site/content/en/docs/handbook/pushing.md
+++ b/site/content/en/docs/handbook/pushing.md
@@ -136,7 +136,7 @@ eval $(minikube podman-env)
 You should now be able to use podman client on the command line on your host machine talking to the podman service inside the minikube VM:
 
 {{% tabs %}}
-{{% tab "Linux" %}}
+{{% linuxtab %}}
 
 ```shell
 podman-remote help
@@ -146,8 +146,8 @@ podman-remote help
 Note: On Linux the remote client is called "podman-remote", while the local program is called "podman".
 {{% /pageinfo %}}
 
-{{% /tab %}}
-{{% tab "macOS" %}}
+{{% /linuxtab %}}
+{{% mactab %}}
 
 ```shell
 podman help
@@ -157,8 +157,8 @@ podman help
 Note: On macOS the remote client is called "podman", since there is no local "podman" program available.
 {{% /pageinfo %}}
 
-{{% /tab %}}
-{{% tab "Windows" %}}
+{{% /mactab %}}
+{{% windowstab %}}
 
 ```shell
 podman help
@@ -168,7 +168,7 @@ podman help
 Note: On Windows the remote client is called "podman", since there is no local "podman" program available.
 {{% /pageinfo %}}
 
-{{% /tab %}}
+{{% /windowstab %}}
 {{% /tabs %}}
 
 
diff --git a/site/content/en/docs/start/_index.md b/site/content/en/docs/start/_index.md
index d3ede45804d5..b15b010c7bca 100644
--- a/site/content/en/docs/start/_index.md
+++ b/site/content/en/docs/start/_index.md
@@ -22,7 +22,7 @@ All you need is Docker (or similarly compatible) container or a Virtual Machine
 <h2 class="step"><span class="fa-stack fa-1x"><i class="fa fa-circle fa-stack-2x"></i><strong class="fa-stack-1x text-primary">1</strong></span>Installation</h2>
 
 {{% tabs %}}
-{{% tab "Linux" %}}
+{{% linuxtab %}}
 
 For Linux users, we provide 3 easy download options:
 
@@ -47,8 +47,8 @@ curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-latest
 sudo rpm -ivh minikube-latest.x86_64.rpm
 ```
 
-{{% /tab %}}
-{{% tab "macOS" %}}
+{{% /linuxtab %}}
+{{% mactab %}}
 
 If the [Brew Package Manager](https://brew.sh/) installed:
 
@@ -70,8 +70,8 @@ curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-darwin
 sudo install minikube-darwin-amd64 /usr/local/bin/minikube
 ```
 
-{{% /tab %}}
-{{% tab "Windows" %}}
+{{% /mactab %}}
+{{% windowstab %}}
 
 If the [Chocolatey Package Manager](https://chocolatey.org/) is installed, use it to install minikube:
 
@@ -81,7 +81,7 @@ choco install minikube
 
 Otherwise, download and run the [Windows installer](https://storage.googleapis.com/minikube/releases/latest/minikube-installer.exe)
 
-{{% /tab %}}
+{{% /windowstab %}}
 {{% /tabs %}}
 
 <h2 class="step"><span class="fa-stack fa-1x"><i class="fa fa-circle fa-stack-2x"></i><strong class="fa-stack-1x text-primary">2</strong></span>Start your cluster</h2>
diff --git a/site/layouts/shortcodes/linuxtab.html b/site/layouts/shortcodes/linuxtab.html
new file mode 100644
index 000000000000..2e229e494920
--- /dev/null
+++ b/site/layouts/shortcodes/linuxtab.html
@@ -0,0 +1,3 @@
+<div class="tab-pane Linux" title="Linux" os="Linux">
+    {{ .Inner }}
+</div>
diff --git a/site/layouts/shortcodes/mactab.html b/site/layouts/shortcodes/mactab.html
new file mode 100644
index 000000000000..7e8d3c75ebd1
--- /dev/null
+++ b/site/layouts/shortcodes/mactab.html
@@ -0,0 +1,3 @@
+<div class="tab-pane Mac" title="macOS" os="Mac">
+    {{ .Inner }}
+</div>
diff --git a/site/layouts/shortcodes/windowstab.html b/site/layouts/shortcodes/windowstab.html
new file mode 100644
index 000000000000..bb9e9a1ada4f
--- /dev/null
+++ b/site/layouts/shortcodes/windowstab.html
@@ -0,0 +1,3 @@
+<div class="tab-pane Windows" title="Windows" os="Windows">
+    {{ .Inner }}
+</div>
diff --git a/site/static/js/tabs.js b/site/static/js/tabs.js
index 06ae6a5562b1..88800d84d0e5 100644
--- a/site/static/js/tabs.js
+++ b/site/static/js/tabs.js
@@ -2,18 +2,21 @@
 function initTabs() {
     $('.tab-content').find('.tab-pane').each(function(idx, item) {
       var navTabs = $(this).closest('.code-tabs').find('.nav-tabs'),
-          title = $(this).attr('title');
-      navTabs.append('<li class="nav-tab"><a href="#" class="nav-tab">'+title+'</a></li>');
+          title = $(this).attr('title'),
+          os = $(this).attr('os');
+      navTabs.append('<li class="nav-tab '+os+'"><a href="#" class="nav-tab">'+title+'</a></li>');
     });
-   
+
     $('.code-tabs ul.nav-tabs').each(function() {
-      $(this).find("li:first").addClass('active');
-    })
-  
-    $('.code-tabs .tab-content').each(function() {
-      $(this).find("div:first").addClass('active');
+      let tabSelector = getTabSelector(this);
+      $(this).find('li'+tabSelector).addClass('active');
     });
-  
+
+    $('.code-tabs .tab-content').each(function() {
+      let tabSelector = getTabSelector(this);
+      $(this).find('div'+tabSelector).addClass('active');
+    })
+ 
     $('.nav-tabs a').click(function(e){
       e.preventDefault();
       var tab = $(this).parent(),
@@ -25,3 +28,18 @@ function initTabs() {
       tabPane.addClass('active');
     });
   }
+
+const getTabSelector = currElement => {
+    let osSelector = '.'+getUserOS();
+    let hasMatchingOSTab = $(currElement).find(osSelector).length;
+    return hasMatchingOSTab ? osSelector : ':first';
+}
+
+const getUserOS = () => {
+    let os = ['Linux', 'Mac', 'Windows'];
+    let userAgent = navigator.userAgent;
+    for (let currentOS of os) {
+      if (userAgent.indexOf(currentOS) !== -1) return currentOS;
+    }
+    return 'Linux';
+}

From e1aa07feb2e878bdf6719b1740724219b29b3f55 Mon Sep 17 00:00:00 2001
From: msedzins <msedzins@gmail.com>
Date: Mon, 30 Nov 2020 22:53:27 +0000
Subject: [PATCH 17/45] Ability to use a custom TLS certificate with the
 Ingress Tutorial added

---
 cmd/minikube/cmd/config/configure.go          | 17 +++++++
 deploy/addons/ingress/ingress-dp.yaml.tmpl    |  3 ++
 pkg/minikube/assets/addons.go                 |  2 +
 pkg/minikube/config/types.go                  |  1 +
 .../en/docs/tutorials/custom_cert_ingress.md  | 44 +++++++++++++++++++
 5 files changed, 67 insertions(+)
 create mode 100644 site/content/en/docs/tutorials/custom_cert_ingress.md

diff --git a/cmd/minikube/cmd/config/configure.go b/cmd/minikube/cmd/config/configure.go
index 5b4d5301b6bb..b5800cf2fb84 100644
--- a/cmd/minikube/cmd/config/configure.go
+++ b/cmd/minikube/cmd/config/configure.go
@@ -19,6 +19,7 @@ package config
 import (
 	"io/ioutil"
 	"net"
+	"regexp"
 
 	"github.com/spf13/cobra"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -204,6 +205,22 @@ var addonsConfigureCmd = &cobra.Command{
 				cfg.KubernetesConfig.LoadBalancerEndIP = AskForStaticValidatedValue("-- Enter Load Balancer End IP: ", validator)
 			}
 
+			if err := config.SaveProfile(profile, cfg); err != nil {
+				out.ErrT(style.Fatal, "Failed to save config {{.profile}}", out.V{"profile": profile})
+			}
+		case "ingress":
+			profile := ClusterFlagValue()
+			_, cfg := mustload.Partial(profile)
+
+			validator := func(s string) bool {
+				format := regexp.MustCompile("^.+/.+$")
+				return format.MatchString(s)
+			}
+
+			if cfg.KubernetesConfig.CustomIngressCert == "" {
+				cfg.KubernetesConfig.CustomIngressCert = AskForStaticValidatedValue("-- Enter custom cert(format is \"namespace/secret\"): ", validator)
+			}
+
 			if err := config.SaveProfile(profile, cfg); err != nil {
 				out.ErrT(style.Fatal, "Failed to save config {{.profile}}", out.V{"profile": profile})
 			}
diff --git a/deploy/addons/ingress/ingress-dp.yaml.tmpl b/deploy/addons/ingress/ingress-dp.yaml.tmpl
index 56f9dc344373..fda40ff2bae0 100644
--- a/deploy/addons/ingress/ingress-dp.yaml.tmpl
+++ b/deploy/addons/ingress/ingress-dp.yaml.tmpl
@@ -65,6 +65,9 @@ spec:
             - --validating-webhook=:8443
             - --validating-webhook-certificate=/usr/local/certificates/cert
             - --validating-webhook-key=/usr/local/certificates/key
+            {{if .CustomIngressCert}}
+            - --default-ssl-certificate={{ .CustomIngressCert }} 
+           {{end}}
           securityContext:
             capabilities:
               drop:
diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
index 609b10381838..85aa09d225e5 100644
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@@ -489,6 +489,7 @@ func GenerateTemplateData(cfg config.KubernetesConfig) interface{} {
 		ImageRepository           string
 		LoadBalancerStartIP       string
 		LoadBalancerEndIP         string
+		CustomIngressCert         string
 		StorageProvisionerVersion string
 	}{
 		Arch:                      a,
@@ -496,6 +497,7 @@ func GenerateTemplateData(cfg config.KubernetesConfig) interface{} {
 		ImageRepository:           cfg.ImageRepository,
 		LoadBalancerStartIP:       cfg.LoadBalancerStartIP,
 		LoadBalancerEndIP:         cfg.LoadBalancerEndIP,
+		CustomIngressCert:         cfg.CustomIngressCert,
 		StorageProvisionerVersion: version.GetStorageProvisionerVersion(),
 	}
 
diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index 1d19604bc4af..d95e9f7c85f4 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -92,6 +92,7 @@ type KubernetesConfig struct {
 	ImageRepository     string
 	LoadBalancerStartIP string // currently only used by MetalLB addon
 	LoadBalancerEndIP   string // currently only used by MetalLB addon
+	CustomIngressCert   string // used by Ingress addon
 	ExtraOptions        ExtraOptionSlice
 
 	ShouldLoadCachedImages bool
diff --git a/site/content/en/docs/tutorials/custom_cert_ingress.md b/site/content/en/docs/tutorials/custom_cert_ingress.md
new file mode 100644
index 000000000000..1f419e6207e7
--- /dev/null
+++ b/site/content/en/docs/tutorials/custom_cert_ingress.md
@@ -0,0 +1,44 @@
+---
+title: "How to use custom TLS certificate with ingress addon"
+linkTitle: "Using custom TLS certificate with ingress addon"
+weight: 1
+date: 2020-11-30
+---
+
+## Overview
+
+- This tutorial will show you how to configure custom TLS certificatate for ingress addon.
+
+## Tutorial
+
+- Start minikube
+```
+$ minikube start
+```
+
+- Create TLS secret which contains custom certificate and private key
+```
+$ kubectl -n kube-system create secret tls mkcert --key key.pem --cert cert.pem
+```
+
+- Configure ingress addon
+```
+$ minikube addons configure ingress
+-- Enter custom cert(format is "namespace/secret"): kube-system/mkcert
+✅  ingress was successfully configured
+```
+
+- Enable ingress addon (disable first when already enabled)
+```
+$ minikube addons disable ingress
+🌑  "The 'ingress' addon is disabled
+
+$ minikube addons enable ingress
+🔎  Verifying ingress addon...
+🌟  The 'ingress' addon is enabled
+```
+- Verify if custom certificate was enabled
+```
+$ kubectl -n kube-system get deployment ingress-nginx-controller -o yaml | grep "kube-system"
+- --default-ssl-certificate=kube-system/mkcert
+```
\ No newline at end of file

From 2a11844320af6d827b0573c40d56af76b5373bec Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 30 Nov 2020 17:01:15 -0800
Subject: [PATCH 18/45] Only rebuild latest if commit has changed

Also store commit in the metadata of binaries in GCS so it's easy to figure out what commit they were built at.
---
 hack/jenkins/build/minikube_build_upload_latest.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/hack/jenkins/build/minikube_build_upload_latest.sh b/hack/jenkins/build/minikube_build_upload_latest.sh
index b311e0e8ad04..b6b50cc86ce7 100755
--- a/hack/jenkins/build/minikube_build_upload_latest.sh
+++ b/hack/jenkins/build/minikube_build_upload_latest.sh
@@ -28,9 +28,18 @@ WANT_GOLANG_VERSION=$(grep '^GO_VERSION' Makefile | awk '{ print $3 }')
 
 declare -rx GOPATH=/var/lib/jenkins/go
 
+GIT_COMMIT_AT_HEAD=$(git rev-parse HEAD | xargs)
+MINIKUBE_LATEST_COMMIT=$(gsutil stat gs://minikube/latest/minikube-linux-amd64 | grep commit | awk '{ print $2 }' | xargs)
+
+if [ "$GIT_COMMIT_AT_HEAD" = "$MINIKUBE_LATEST_COMMIT" ]; then
+  echo "The current uploaded binary is already latest, skipping build"
+  exit 0
+fi
+
 make cross && failed=$? || failed=$?
 if [[ "${failed}" -ne 0 ]]; then
   echo "build failed"
   exit "${failed}"
 fi
 gsutil cp out/minikube-* "gs://${bucket}"
+gsutil setmeta -r -h "x-goog-meta-commit:$GIT_COMMIT_AT_HEAD" "gs://${bucket}"

From 8314c631f547497459dbde129ad3e2b916488f32 Mon Sep 17 00:00:00 2001
From: edtrist <edtrist@users.noreply.github.com>
Date: Tue, 1 Dec 2020 02:02:08 +0000
Subject: [PATCH 19/45] Update Docsy theme to latest version

---
 site/themes/docsy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/themes/docsy b/site/themes/docsy
index dd303fd19cc1..2536303cad19 160000
--- a/site/themes/docsy
+++ b/site/themes/docsy
@@ -1 +1 @@
-Subproject commit dd303fd19cc13ffc01bcbe86ff54f21e423d04de
+Subproject commit 2536303cad19991c673037f4f16332075141364a

From ae4fa9a2ee25a11e1c125987dcd5b1d9c839fee2 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 30 Nov 2020 19:03:30 -0800
Subject: [PATCH 20/45] Address code review comments`

---
 hack/metrics/metrics.go  | 26 +++++++++++-----
 hack/metrics/minikube.go | 67 +++++++++++++++++++++++++++-------------
 2 files changed, 65 insertions(+), 28 deletions(-)

diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go
index 363511b6a8b3..cd0c2ded935f 100644
--- a/hack/metrics/metrics.go
+++ b/hack/metrics/metrics.go
@@ -19,6 +19,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -97,8 +98,20 @@ func execute() error {
 	}
 	defer sd.StopMetricsExporter()
 	ctx := context.Background()
+
+	// create a tmp file to download minikube to
+	tmp, err := ioutil.TempFile("", binary())
+	if err != nil {
+		return errors.Wrap(err, "creating tmp file")
+	}
+	if err := tmp.Close(); err != nil {
+		return errors.Wrap(err, "closing file")
+	}
+	defer os.Remove(tmp.Name())
+
+	// start loop where we will track minikube start time
 	for {
-		st, err := minikubeStartTime(ctx, projectID)
+		st, err := minikubeStartTime(ctx, projectID, tmp.Name())
 		if err != nil {
 			log.Printf("error collecting start time: %v", err)
 			continue
@@ -109,12 +122,10 @@ func execute() error {
 	}
 }
 
-func minikubeStartTime(ctx context.Context, projectID string) (float64, error) {
-	minikubePath, err := downloadMinikube()
-	if err != nil {
+func minikubeStartTime(ctx context.Context, projectID, minikubePath string) (float64, error) {
+	if err := downloadMinikube(ctx, minikubePath); err != nil {
 		return 0, errors.Wrap(err, "downloading minikube")
 	}
-	defer os.Remove(minikubePath)
 	defer deleteMinikube(ctx, minikubePath)
 
 	cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile, "--memory=2000", "--trace=gcp")
@@ -123,11 +134,12 @@ func minikubeStartTime(ctx context.Context, projectID string) (float64, error) {
 	cmd.Stderr = os.Stderr
 
 	t := time.Now()
-	log.Print("Running minikube start....")
+	log.Printf("Running [%v]....", cmd.Args)
 	if err := cmd.Run(); err != nil {
 		return 0, errors.Wrapf(err, "running %v", cmd.Args)
 	}
-	return time.Since(t).Seconds(), nil
+	totalTime := time.Since(t).Seconds()
+	return totalTime, nil
 }
 
 func deleteMinikube(ctx context.Context, minikubePath string) {
diff --git a/hack/metrics/minikube.go b/hack/metrics/minikube.go
index 16086873e58d..d8c6149ea761 100644
--- a/hack/metrics/minikube.go
+++ b/hack/metrics/minikube.go
@@ -21,7 +21,9 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"runtime"
+	"strings"
 
 	"log"
 
@@ -33,39 +35,62 @@ const (
 	bucketName = "minikube/latest"
 )
 
-// download minikube latest to a tmp file
-func downloadMinikube() (string, error) {
-	b := binary()
-	tmp, err := ioutil.TempFile("", b)
+// download minikube latest to file
+func downloadMinikube(ctx context.Context, minikubePath string) error {
+	client, err := storage.NewClient(ctx)
 	if err != nil {
-		return "", errors.Wrap(err, "creating tmp file")
+		return errors.Wrap(err, "creating client")
 	}
-	if err := tmp.Close(); err != nil {
-		return "", errors.Wrap(err, "closing tmp file")
-	}
-	client, err := storage.NewClient(context.Background())
-	if err != nil {
-		return "", errors.Wrap(err, "creating client")
+
+	if localMinikubeIsLatest(ctx, minikubePath, client) {
+		log.Print("local minikube is latest, skipping download...")
+		return nil
 	}
-	ctx := context.Background()
-	rc, err := client.Bucket(bucketName).Object(b).NewReader(ctx)
+
+	os.Remove(minikubePath)
+	// download minikube binary from GCS
+	obj := client.Bucket("minikube").Object(fmt.Sprintf("latest/%s", binary()))
+	rc, err := obj.NewReader(ctx)
 	if err != nil {
-		return "", errors.Wrap(err, "gcs new reader")
+		return errors.Wrap(err, "gcs new reader")
 	}
 	defer rc.Close()
 
 	data, err := ioutil.ReadAll(rc)
 	if err != nil {
-		return "", errors.Wrap(err, "ioutil read all")
+		return errors.Wrap(err, "ioutil read all")
+	}
+	log.Printf("downloading gs://%s/%s to %v", bucketName, binary(), minikubePath)
+	if err := ioutil.WriteFile(minikubePath, data, 0777); err != nil {
+		return errors.Wrap(err, "writing minikubePath")
 	}
-	log.Printf("downloading gs://%s/%s to %v", bucketName, b, tmp.Name())
-	if err := ioutil.WriteFile(tmp.Name(), data, 0777); err != nil {
-		return "", errors.Wrap(err, "writing file")
+	if err := os.Chmod(minikubePath, 0700); err != nil {
+		return errors.Wrap(err, "chmod")
 	}
-	if err := os.Chmod(tmp.Name(), 0700); err != nil {
-		return "", errors.Wrap(err, "chmod")
+	return nil
+}
+
+// localMinikubeIsLatest returns true if the local version of minikube
+// matches the latest version in GCS
+func localMinikubeIsLatest(ctx context.Context, minikubePath string, client *storage.Client) bool {
+	log.Print("checking if local minikube is latest...")
+	obj := client.Bucket("minikube").Object(fmt.Sprintf("latest/%s", binary()))
+	attrs, err := obj.Attrs(ctx)
+	if err != nil {
+		log.Printf("error getting %s object attrs: %v", obj.ObjectName(), err)
+		return false
+	}
+	gcsMinikubeVersion, ok := attrs.Metadata["commit"]
+	if !ok {
+		log.Printf("there is no commit: %v", attrs.Metadata)
+		return false
+	}
+	currentMinikubeVersion, err := exec.Command(minikubePath, "version", "--output=json").Output()
+	if err != nil {
+		log.Printf("error running [%s version]: %v", minikubePath, err)
+		return false
 	}
-	return tmp.Name(), nil
+	return strings.Contains(string(currentMinikubeVersion), gcsMinikubeVersion)
 }
 
 func binary() string {

From 8590560f838617b8ce6bfb6601bdd16ec0632f52 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 30 Nov 2020 19:21:36 -0800
Subject: [PATCH 21/45] use same storage.ObjectHandle for downloading file and
 getting commmit

---
 hack/metrics/minikube.go | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/hack/metrics/minikube.go b/hack/metrics/minikube.go
index d8c6149ea761..5b9d64920540 100644
--- a/hack/metrics/minikube.go
+++ b/hack/metrics/minikube.go
@@ -32,7 +32,7 @@ import (
 )
 
 const (
-	bucketName = "minikube/latest"
+	bucketName = "minikube"
 )
 
 // download minikube latest to file
@@ -41,15 +41,15 @@ func downloadMinikube(ctx context.Context, minikubePath string) error {
 	if err != nil {
 		return errors.Wrap(err, "creating client")
 	}
+	obj := client.Bucket("minikube").Object(fmt.Sprintf("latest/%s", binary()))
 
-	if localMinikubeIsLatest(ctx, minikubePath, client) {
+	if localMinikubeIsLatest(ctx, minikubePath, obj) {
 		log.Print("local minikube is latest, skipping download...")
 		return nil
 	}
 
 	os.Remove(minikubePath)
 	// download minikube binary from GCS
-	obj := client.Bucket("minikube").Object(fmt.Sprintf("latest/%s", binary()))
 	rc, err := obj.NewReader(ctx)
 	if err != nil {
 		return errors.Wrap(err, "gcs new reader")
@@ -72,9 +72,8 @@ func downloadMinikube(ctx context.Context, minikubePath string) error {
 
 // localMinikubeIsLatest returns true if the local version of minikube
 // matches the latest version in GCS
-func localMinikubeIsLatest(ctx context.Context, minikubePath string, client *storage.Client) bool {
+func localMinikubeIsLatest(ctx context.Context, minikubePath string, obj *storage.ObjectHandle) bool {
 	log.Print("checking if local minikube is latest...")
-	obj := client.Bucket("minikube").Object(fmt.Sprintf("latest/%s", binary()))
 	attrs, err := obj.Attrs(ctx)
 	if err != nil {
 		log.Printf("error getting %s object attrs: %v", obj.ObjectName(), err)

From ee1da89f11cbe42b295c1081c52436368eaebf5d Mon Sep 17 00:00:00 2001
From: edtrist <edtrist@users.noreply.github.com>
Date: Tue, 1 Dec 2020 18:25:41 +0000
Subject: [PATCH 22/45] Enable Prism code highlighting / copy buttons

---
 site/config.toml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/site/config.toml b/site/config.toml
index 7a9c6cf08704..3e6c1da9dbb4 100644
--- a/site/config.toml
+++ b/site/config.toml
@@ -92,6 +92,9 @@ github_subdir = "site"
 # enabling local search https://www.docsy.dev/docs/adding-content/navigation/#configure-local-search-with-lunr
 offlineSearch = true
 
+# Enable syntax highlighting and copy buttons on code blocks with Prism
+prism_syntax_highlighting = true
+
 # User interface configuration
 [params.ui]
 # Enable to show the side bar menu in its compact state.

From aa64b8bf7fec5daa816ba368225cd8c4ef0a945f Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 1 Dec 2020 11:47:23 -0800
Subject: [PATCH 23/45] bump gopogh

---
 .github/workflows/iso.yml       |  2 +-
 .github/workflows/kic_image.yml |  2 +-
 .github/workflows/master.yml    | 22 +++++++++++-----------
 .github/workflows/pr.yml        | 22 +++++++++++-----------
 hack/jenkins/common.sh          |  4 ++--
 5 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/iso.yml b/.github/workflows/iso.yml
index 07e03d719712..d06a930eee58 100644
--- a/.github/workflows/iso.yml
+++ b/.github/workflows/iso.yml
@@ -82,7 +82,7 @@ jobs:
       - name: Install gopogh
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
           sudo apt-get install -y jq
       - name: Run Integration Test
diff --git a/.github/workflows/kic_image.yml b/.github/workflows/kic_image.yml
index b3ddee81b208..cadf209cabc3 100644
--- a/.github/workflows/kic_image.yml
+++ b/.github/workflows/kic_image.yml
@@ -60,7 +60,7 @@ jobs:
       - name: Install gopogh
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
           sudo apt-get install -y jq
           rm -f gopogh-linux-amd64 || true
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 73d297e07edf..d89d6156bf9b 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -120,7 +120,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -205,7 +205,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Install docker
         shell: bash
@@ -350,7 +350,7 @@ jobs:
         continue-on-error: true
         shell: powershell
         run: |
-          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
+          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
           choco install -y kubernetes-cli
           choco install -y jq
           choco install -y caffeine
@@ -487,7 +487,7 @@ jobs:
         shell: powershell
         run: |
           $ErrorActionPreference = "SilentlyContinue"
-          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
+          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
           choco install -y kubernetes-cli
           choco install -y jq
           choco install -y caffeine
@@ -592,7 +592,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -689,7 +689,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -771,7 +771,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Install docker
         shell: bash
@@ -883,7 +883,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -967,7 +967,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -1074,7 +1074,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -1156,7 +1156,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 8e0021e66369..c84945b0df02 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -118,7 +118,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -203,7 +203,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Install docker
         shell: bash
@@ -348,7 +348,7 @@ jobs:
         continue-on-error: true
         shell: powershell
         run: |
-          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
+          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
           choco install -y kubernetes-cli
           choco install -y jq
           choco install -y caffeine
@@ -485,7 +485,7 @@ jobs:
         shell: powershell
         run: |
           $ErrorActionPreference = "SilentlyContinue"
-          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
+          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
           choco install -y kubernetes-cli
           choco install -y jq
           choco install -y caffeine
@@ -590,7 +590,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -687,7 +687,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -769,7 +769,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Install docker
         shell: bash
@@ -881,7 +881,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -965,7 +965,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -1072,7 +1072,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64
           sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
@@ -1154,7 +1154,7 @@ jobs:
 
         shell: bash
         run: |
-          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64
+          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64
           sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
       - name: Download Binaries
         uses: actions/download-artifact@v1
diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh
index 0f8cec2a2093..aebdb5cea000 100755
--- a/hack/jenkins/common.sh
+++ b/hack/jenkins/common.sh
@@ -347,9 +347,9 @@ fi
 
 echo ">> Installing gopogh"
 if [ "$(uname)" != "Darwin" ]; then
-  curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-linux-amd64 && sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
+  curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-linux-amd64 && sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
 else
-  curl -LO https://github.com/medyagh/gopogh/releases/download/v0.2.4/gopogh-darwin-amd64 && sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
+  curl -LO https://github.com/medyagh/gopogh/releases/download/v0.3.0/gopogh-darwin-amd64 && sudo install gopogh-darwin-amd64 /usr/local/bin/gopogh
 fi
 
 echo ">> Running gopogh"

From d1410363820008be447956431d6fa91f53cf4030 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Tue, 1 Dec 2020 14:29:09 -0800
Subject: [PATCH 24/45] bump storage provsioner to multi arch

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 836d7f78b89c..3d522f8a68ba 100644
--- a/Makefile
+++ b/Makefile
@@ -92,7 +92,7 @@ SHA512SUM=$(shell command -v sha512sum || echo "shasum -a 512")
 GVISOR_TAG ?= latest
 
 # storage provisioner tag to push changes to
-STORAGE_PROVISIONER_TAG ?= v3
+STORAGE_PROVISIONER_TAG ?= v4
 
 STORAGE_PROVISIONER_MANIFEST ?= $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG)
 STORAGE_PROVISIONER_IMAGE ?= $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG)

From 2c6dc34f04535dfb19022f46f780b526db3dfed3 Mon Sep 17 00:00:00 2001
From: edtrist <edtrist@users.noreply.github.com>
Date: Tue, 1 Dec 2020 19:30:51 +0000
Subject: [PATCH 25/45] Styling updates after Docsy update

---
 site/assets/scss/_variables_project.scss  | 16 +++++++++++++++-
 site/layouts/partials/hooks/head-end.html |  2 +-
 site/layouts/partials/sidebar-tree.html   |  2 +-
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/site/assets/scss/_variables_project.scss b/site/assets/scss/_variables_project.scss
index 7bc0efdb2206..536b3b4a01f4 100644
--- a/site/assets/scss/_variables_project.scss
+++ b/site/assets/scss/_variables_project.scss
@@ -180,7 +180,7 @@ section.td-box--height-auto {
 // Allow code tags to span most of a window length (default is 80%)
 pre {
     max-width: 99% !important;
-    font-family: 'Inconsolata', monospace !important;
+    font-family: 'SFMono-Regular', Menlo, Monaco, Consolas, 'liberation mono', 'courier new', monospace !important;
     font-size: 13px !important;
 }
 
@@ -197,6 +197,17 @@ div.td-content {
         padding: 0.5em !important;
         margin-top: 1.25em;
         margin-bottom: 1.25em;
+        background-color: $gray-100;
+        max-width: 99% !important;
+
+        pre {
+            background-color: inherit !important;
+            padding: 0 !important;
+
+            code {
+                font-family: inherit !important;
+            }
+        }
     }
 }
 
@@ -214,3 +225,6 @@ div.td-content {
     max-width: 460px;
 }
 
+div.code-toolbar > .toolbar {
+    top: -.3em !important;
+}
diff --git a/site/layouts/partials/hooks/head-end.html b/site/layouts/partials/hooks/head-end.html
index ed11ad10a1ff..bf082de3bd90 100644
--- a/site/layouts/partials/hooks/head-end.html
+++ b/site/layouts/partials/hooks/head-end.html
@@ -1,5 +1,5 @@
 <!-- start: minikube override head-end partial -->
-<link href="https://fonts.googleapis.com/css2?family=Inconsolata&family=Lora&family=Open+Sans:wght@600;700&display=swap" rel="stylesheet">
+<link href="https://fonts.googleapis.com/css2?family=Lora&family=Open+Sans:wght@600;700&display=swap" rel="stylesheet">
 <link href="/css/tabs.css" rel="stylesheet">
 <script src="/js/tabs.js"></script>
 <!-- end: minikube override head-end partial -->
\ No newline at end of file
diff --git a/site/layouts/partials/sidebar-tree.html b/site/layouts/partials/sidebar-tree.html
index 029abae4c55e..b1527f944fc0 100644
--- a/site/layouts/partials/sidebar-tree.html
+++ b/site/layouts/partials/sidebar-tree.html
@@ -12,7 +12,7 @@
     </button>
   </form>
   {{ end }}
-  <nav class="collapse td-sidebar-nav pt-2 pl-4" id="td-section-nav">
+  <nav class="collapse td-sidebar-nav" id="td-section-nav">
     {{ if  (gt (len .Site.Home.Translations) 0) }}
     <div class="nav-item dropdown d-block d-lg-none">
       {{ partial "navbar-lang-selector.html" . }}

From f6a770e0a605048439d498fabc29a570c0628690 Mon Sep 17 00:00:00 2001
From: edtrist <edtrist@users.noreply.github.com>
Date: Tue, 1 Dec 2020 20:09:16 +0000
Subject: [PATCH 26/45] Add missing language to code blocks

---
 site/content/en/docs/contrib/addons.en.md     | 20 +++++++---
 .../en/docs/contrib/documentation.en.md       |  8 +++-
 .../content/en/docs/contrib/json_output.en.md |  4 +-
 .../en/docs/contrib/releasing/gvisor.md       |  4 +-
 site/content/en/docs/faq/_index.md            |  9 +++--
 site/content/en/docs/handbook/accessing.md    | 20 ++++++----
 .../en/docs/handbook/addons/gcp-auth.md       | 17 ++++++--
 site/content/en/docs/handbook/config.md       |  4 +-
 site/content/en/docs/handbook/controls.md     | 40 ++++++++++++++-----
 site/content/en/docs/handbook/deploying.md    |  2 +-
 site/content/en/docs/handbook/filesync.md     |  2 +-
 site/content/en/docs/handbook/kubectl.md      | 24 ++++++++---
 site/content/en/docs/handbook/mount.md        |  4 +-
 site/content/en/docs/handbook/registry.md     | 14 +++----
 .../en/docs/handbook/troubleshooting.md       |  9 +++--
 .../en/docs/handbook/untrusted_certs.md       |  2 +-
 site/content/en/docs/start/_index.md          |  6 ++-
 .../ambassador_ingress_controller.md          | 12 ++++--
 site/content/en/docs/tutorials/multi_node.md  | 37 ++++++++++++-----
 site/content/en/docs/tutorials/telemetry.md   |  2 +-
 site/content/en/docs/tutorials/using_psp.md   | 14 +++++--
 21 files changed, 179 insertions(+), 75 deletions(-)

diff --git a/site/content/en/docs/contrib/addons.en.md b/site/content/en/docs/contrib/addons.en.md
index b4325e4eafdf..1305f88a59ef 100644
--- a/site/content/en/docs/contrib/addons.en.md
+++ b/site/content/en/docs/contrib/addons.en.md
@@ -9,23 +9,33 @@ description: >
 
 To create an addon, first fork the minikube repository, and check out your fork:
 
-`git clone git@github.com:<username>/minikube.git`
+```shell
+git clone git@github.com:<username>/minikube.git
+```
 
 Then go into the source directory:
 
-`cd minikube`
+```shell
+cd minikube
+```
 
 Create a subdirectory:
 
-`mkdir deploy/addons/<addon name>`
+```shell
+mkdir deploy/addons/<addon name>
+```
 
 Add your manifest YAML's to the directory you have created:
 
-`cp *.yaml deploy/addons/<addon name>`
+```shell
+cp *.yaml deploy/addons/<addon name>
+```
 
 Note: If the addon never needs authentication to GCP, then consider adding the following label to the pod's yaml:
 
-`gcp-auth-skip-secret: "true"`
+```yaml
+gcp-auth-skip-secret: "true"
+```
 
 To make the addon appear in `minikube addons list`, add it to `pkg/addons/config.go`. Here is the entry used by the `registry` addon, which will work for any addon which does not require custom code:
 
diff --git a/site/content/en/docs/contrib/documentation.en.md b/site/content/en/docs/contrib/documentation.en.md
index 23e8408c9dcf..2a3036956ba1 100644
--- a/site/content/en/docs/contrib/documentation.en.md
+++ b/site/content/en/docs/contrib/documentation.en.md
@@ -22,7 +22,9 @@ Use Github's repositories and markdown editor as described by [Kubernetes's gene
 
 To serve documentation pages locally, clone the `minikube` repository and run:
 
-`make site`
+```shell
+make site
+```
 
 Notes :
 
@@ -33,7 +35,9 @@ Notes :
 
 We recommend installing [markdownlint](https://github.com/markdownlint/markdownlint) to find issues with your markdown file. Once installed, you can use this handy target:
 
-`make mdlint`
+```shell
+make mdlint
+```
 
 ## Style Guidelines
 
diff --git a/site/content/en/docs/contrib/json_output.en.md b/site/content/en/docs/contrib/json_output.en.md
index 888d819aa3ad..398e356baa4f 100644
--- a/site/content/en/docs/contrib/json_output.en.md
+++ b/site/content/en/docs/contrib/json_output.en.md
@@ -12,7 +12,7 @@ You may need to add logs to the registry if the `TestJSONOutput` integration tes
 ### Background
 minikube provides JSON output for `minikube start`, accesible via the `--output` flag:
 
-```
+```shell
 minikube start --output json
 ```
 
@@ -61,7 +61,7 @@ You will need to add your new log in two places:
 
 Finally, set your new step in the cod by placing this line before you call `out.T`:
 
-```
+```go
 register.Reg.SetStep(register.MyNewStep)
 ```
 
diff --git a/site/content/en/docs/contrib/releasing/gvisor.md b/site/content/en/docs/contrib/releasing/gvisor.md
index 9d3c302c4c28..0b1cec2b475c 100644
--- a/site/content/en/docs/contrib/releasing/gvisor.md
+++ b/site/content/en/docs/contrib/releasing/gvisor.md
@@ -30,4 +30,6 @@ The image is located at `gcr.io/k8s-minikube/gvisor-addon`
 
 ## Updating the gVisor image
 
-`make push-gvisor-addon-image`
+```shell
+make push-gvisor-addon-image
+```
diff --git a/site/content/en/docs/faq/_index.md b/site/content/en/docs/faq/_index.md
index 0539c92f2873..8a495492769d 100644
--- a/site/content/en/docs/faq/_index.md
+++ b/site/content/en/docs/faq/_index.md
@@ -22,15 +22,18 @@ Alternatively, configure `sudo` to never prompt for the commands issued by minik
 
 minikube's bootstrapper, [Kubeadm](https://github.com/kubernetes/kubeadm) verifies a list of features on the host system before installing Kubernetes. in case you get this error, and you still want to try minikube anyways despite your system's limitation you can skip the verification by starting minikube with this extra option:
 
-`minikube start --extra-config kubeadm.ignore-preflight-errors=SystemVerification`
+```shell
+minikube start --extra-config kubeadm.ignore-preflight-errors=SystemVerification
+```
 
 ## what is the resource allocation for Knative Setup using minikube?
 
 Please allocate sufficient resources for Knative setup using minikube, especially when you run a minikube cluster on your local machine. We recommend allocating at least 6 CPUs and 8G memory.
 
-`minikube start --cpus 6 --memory 8000`
+```shell
+minikube start --cpus 6 --memory 8000
+```
 
 ## Do I need to install kubectl locally?
 
 No, minikube comes with built-in kubectl [see minikube's kubectl documentation]({{< ref "docs/handbook/kubectl.md" >}}). 
-
diff --git a/site/content/en/docs/handbook/accessing.md b/site/content/en/docs/handbook/accessing.md
index 6289cc85880a..41c612bca10b 100644
--- a/site/content/en/docs/handbook/accessing.md
+++ b/site/content/en/docs/handbook/accessing.md
@@ -24,7 +24,9 @@ A NodePort service is the most basic way to get external traffic directly to you
 
 We also have a shortcut for fetching the minikube IP and a service's `NodePort`:
 
-`minikube service --url $SERVICE`
+```shell
+minikube service --url $SERVICE
+```
 
 ## Getting the NodePort using kubectl
 
@@ -32,13 +34,17 @@ The minikube VM is exposed to the host system via a host-only IP address, that c
 
 To determine the NodePort for your service, you can use a `kubectl` command like this (note that `nodePort` begins with lowercase `n` in JSON output):
 
-`kubectl get service $SERVICE --output='jsonpath="{.spec.ports[0].nodePort}"'`
+```shell
+kubectl get service $SERVICE --output='jsonpath="{.spec.ports[0].nodePort}"'
+```
 
 ### Increasing the NodePort range
 
 By default, minikube only exposes ports 30000-32767. If this does not work for you, you can adjust the range by using:
 
-`minikube start --extra-config=apiserver.service-node-port-range=1-65535`
+```shell
+minikube start --extra-config=apiserver.service-node-port-range=1-65535
+```
 
 This flag also accepts a comma separated list of ports and port ranges.
 
@@ -57,7 +63,7 @@ Services of type `LoadBalancer` can be exposed via the `minikube tunnel` command
 #### Run tunnel in a separate terminal
 it will ask for password.
 
-```
+```shell
 minikube tunnel
 ```
 
@@ -88,16 +94,16 @@ Status:
 
 
 #### Create a kubernetes deployment 
-```
+```shell
 kubectl create deployment hello-minikube1 --image=k8s.gcr.io/echoserver:1.4
 ```
 #### Create a kubernetes service type LoadBalancer
-```
+```shell
 kubectl expose deployment hello-minikube1 --type=LoadBalancer --port=8080
 ```
 
 ### Check external IP 
-```
+```shell
 kubectl get svc
 ```
 <pre>
diff --git a/site/content/en/docs/handbook/addons/gcp-auth.md b/site/content/en/docs/handbook/addons/gcp-auth.md
index 19aabd3920fe..801a702e9f79 100644
--- a/site/content/en/docs/handbook/addons/gcp-auth.md
+++ b/site/content/en/docs/handbook/addons/gcp-auth.md
@@ -8,8 +8,11 @@ date: 2020-07-15
 If you have a containerized GCP app with a Kubernetes yaml, you can automatically add your credentials to all your deployed pods dynamically with this minikube addon. You just need to have a credentials file, which can be generated with `gcloud auth application-default login`. If you already have a json credentials file you want specify, use the GOOGLE_APPLICATION_CREDENTIALS environment variable.
 
 - Start a cluster:
-```
+```shell
 minikube start
+```
+
+```
 😄  minikube v1.12.0 on Darwin 10.15.5
 ✨  Automatically selected the docker driver. Other choices: hyperkit, virtualbox
 👍  Starting control plane node minikube in cluster minikube
@@ -21,8 +24,11 @@ minikube start
 ```
 
 - Enable the `gcp-auth` addon:
-```
+```shell
 minikube addons enable gcp-auth
+```
+
+```
 🔎  Verifying gcp-auth addon...
 📌  Your GCP credentials will now be mounted into every pod created in the minikube cluster.
 📌  If you don't want credential mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.
@@ -30,14 +36,17 @@ minikube addons enable gcp-auth
 ```
 
 - For credentials in an arbitrary path:
-```
+```shell
 export GOOGLE_APPLICATION_CREDENTIALS=<creds-path>.json 
 minikube addons enable gcp-auth
 ```
 
 - Deploy your GCP app as normal:
-```
+```shell
 kubectl apply -f test.yaml
+```
+
+```
 deployment.apps/pytest created
 ```
 
diff --git a/site/content/en/docs/handbook/config.md b/site/content/en/docs/handbook/config.md
index 9012eb144467..b2168b6850bd 100644
--- a/site/content/en/docs/handbook/config.md
+++ b/site/content/en/docs/handbook/config.md
@@ -49,7 +49,9 @@ This flag is repeated, so you can pass it several times with several different v
 
 By default, minikube installs the latest stable version of Kubernetes that was available at the time of the minikube release. You may select a different Kubernetes release by using the `--kubernetes-version` flag, for example:
 
-`minikube start --kubernetes-version=v1.11.10`
+```shell
+minikube start --kubernetes-version=v1.11.10
+```
   
 minikube follows the [Kubernetes Version and Version Skew Support Policy](https://kubernetes.io/docs/setup/version-skew-policy/), so we guarantee support for the latest build for the last 3 minor Kubernetes releases. When practical, minikube aims to support older releases as well so that users can emulate legacy environments.
 
diff --git a/site/content/en/docs/handbook/controls.md b/site/content/en/docs/handbook/controls.md
index 18866c85009f..baa402de6778 100755
--- a/site/content/en/docs/handbook/controls.md
+++ b/site/content/en/docs/handbook/controls.md
@@ -10,40 +10,60 @@ aliases:
 
 Start a cluster by running:
 
-`minikube start`
+```shell
+minikube start
+```
 
 Access the Kubernetes Dashboard running within the minikube cluster:
 
-`minikube dashboard`
+```shell
+minikube dashboard
+```
 
 Once started, you can interact with your cluster using `kubectl`, just like any other Kubernetes cluster. For instance, starting a server:
 
-`kubectl create deployment hello-minikube --image=k8s.gcr.io/echoserver:1.4`
+```shell
+kubectl create deployment hello-minikube --image=k8s.gcr.io/echoserver:1.4
+```
 
 Exposing a service as a NodePort
 
-`kubectl expose deployment hello-minikube --type=NodePort --port=8080`
+```shell
+kubectl expose deployment hello-minikube --type=NodePort --port=8080
+```
 
 minikube makes it easy to open this exposed endpoint in your browser:
 
-`minikube service hello-minikube`
+```shell
+minikube service hello-minikube
+```
 
 Upgrade your cluster:
 
-`minikube start --kubernetes-version=latest`
+```shell
+minikube start --kubernetes-version=latest
+```
 
 Start a second local cluster (_note: This will not work if minikube is using the bare-metal/none driver_):
 
-`minikube start -p cluster2`
+```shell
+minikube start -p cluster2
+```
 
 Stop your local cluster:
 
-`minikube stop`
+```shell
+minikube stop
+```
 
 Delete your local cluster:
 
-`minikube delete`
+```shell
+minikube delete
+```
 
 Delete all local clusters and profiles
 
-`minikube delete --all`
+```shell
+minikube delete --all
+```
diff --git a/site/content/en/docs/handbook/deploying.md b/site/content/en/docs/handbook/deploying.md
index 521cb949f2cd..701ff2507e57 100644
--- a/site/content/en/docs/handbook/deploying.md
+++ b/site/content/en/docs/handbook/deploying.md
@@ -10,7 +10,7 @@ aliases:
 
 ## kubectl
 
-```
+```shell
 kubectl create deployment hello-minikube1 --image=k8s.gcr.io/echoserver:1.4
 kubectl expose deployment hello-minikube1 --type=LoadBalancer --port=8080
 ```
diff --git a/site/content/en/docs/handbook/filesync.md b/site/content/en/docs/handbook/filesync.md
index 9b83831f484e..355e2ab51a5a 100644
--- a/site/content/en/docs/handbook/filesync.md
+++ b/site/content/en/docs/handbook/filesync.md
@@ -22,7 +22,7 @@ Place files to be synced in `$MINIKUBE_HOME/files`
 
 For example, running the following will result in the deployment of a custom /etc/resolv.conf:
 
-```
+```shell
 mkdir -p ~/.minikube/files/etc
 echo nameserver 8.8.8.8 > ~/.minikube/files/etc/resolv.conf
 minikube start
diff --git a/site/content/en/docs/handbook/kubectl.md b/site/content/en/docs/handbook/kubectl.md
index 24cab2ff661a..e4948013e3f4 100644
--- a/site/content/en/docs/handbook/kubectl.md
+++ b/site/content/en/docs/handbook/kubectl.md
@@ -12,29 +12,41 @@ inside minikube when the `minikube start` command is executed.
 
 However if `kubectl` is not installed locally, minikube already includes kubectl which can be used like this:
 
-`minikube kubectl -- <kubectl commands>`
+```shell
+minikube kubectl -- <kubectl commands>
+```
 
 You can also `alias kubectl="minikube kubectl --"` for easier usage.
 
 Alternatively, you can create a symbolic link to minikube's binary named 'kubectl'.
 
-`ln -s $(which minikube) /usr/local/bin/kubectl`
+```shell
+ln -s $(which minikube) /usr/local/bin/kubectl
+```
 
 Get pods
 
-`minikube kubectl -- get pods`
+```shell
+minikube kubectl -- get pods
+```
 
 Creating a deployment inside kubernetes cluster
 
-`minikube kubectl -- create deployment hello-minikube --image=k8s.gcr.io/echoserver:1.4`
+```shell
+minikube kubectl -- create deployment hello-minikube --image=k8s.gcr.io/echoserver:1.4
+```
 
 Exposing the deployment with a NodePort service
 
-`minikube kubectl -- expose deployment hello-minikube --type=NodePort --port=8080`
+```shell
+minikube kubectl -- expose deployment hello-minikube --type=NodePort --port=8080
+```
 
 For more help
 
-`minikube kubectl -- --help`
+```shell
+minikube kubectl -- --help
+```
 
 ### Shell autocompletion
 
diff --git a/site/content/en/docs/handbook/mount.md b/site/content/en/docs/handbook/mount.md
index 40033f5bcdd0..b0f548c8c922 100644
--- a/site/content/en/docs/handbook/mount.md
+++ b/site/content/en/docs/handbook/mount.md
@@ -14,13 +14,13 @@ aliases:
 
 To mount a directory from the host into the guest using the `mount` subcommand:
 
-```
+```shell
 minikube mount <source directory>:<target directory>
 ```
 
 For example, this would mount your home directory to appear as /host within the minikube VM:
 
-```
+```shell
 minikube mount $HOME:/host
 ```
 
diff --git a/site/content/en/docs/handbook/registry.md b/site/content/en/docs/handbook/registry.md
index 0b04a6f3644f..2a63905b9de7 100644
--- a/site/content/en/docs/handbook/registry.md
+++ b/site/content/en/docs/handbook/registry.md
@@ -50,7 +50,7 @@ Quick guide for configuring minikube and docker on macOS, enabling docker to pus
 
 The first step is to enable the registry addon:
 
-```
+```shell
 minikube addons enable registry
 ```
 
@@ -58,13 +58,13 @@ When enabled, the registry addon exposes its port 5000 on the minikube's virtual
 
 In order to make docker accept pushing images to this registry, we have to redirect port 5000 on the docker virtual machine over to port 5000 on the minikube machine. We can (ab)use docker's network configuration to instantiate a container on the docker's host, and run socat there:
 
-```
+```shell
 docker run --rm -it --network=host alpine ash -c "apk add socat && socat TCP-LISTEN:5000,reuseaddr,fork TCP:$(minikube ip):5000"
 ```
 
 Once socat is running it's possible to push images to the minikube registry:
 
-```
+```shell
 docker tag my/image localhost:5000/myimage
 docker push localhost:5000/myimage
 ```
@@ -77,7 +77,7 @@ Quick guide for configuring minikube and docker on Windows, enabling docker to p
 
 The first step is to enable the registry addon:
 
-```
+```shell
 minikube addons enable registry
 ```
 
@@ -86,7 +86,7 @@ When enabled, the registry addon exposes its port 5000 on the minikube's virtual
 In order to make docker accept pushing images to this registry, we have to redirect port 5000 on the docker virtual machine over to port 5000 on the minikube machine. Unfortunately, the docker vm cannot directly see the IP address of the minikube vm. To fix this, you will have to add one more level of redirection. 
 
 Use kubectl port-forward to map your local workstation to the minikube vm
-```
+```shell
 kubectl port-forward --namespace kube-system <name of the registry vm> 5000:5000
 ```
 
@@ -94,13 +94,13 @@ On your local machine you should now be able to reach the minikube registry by u
 
 From this point we can (ab)use docker's network configuration to instantiate a container on the docker's host, and run socat there to redirect traffic going to the docker vm's port 5000 to port 5000 on your host workstation.
 
-```
+```shell
 docker run --rm -it --network=host alpine ash -c "apk add socat && socat TCP-LISTEN:5000,reuseaddr,fork TCP:host.docker.internal:5000"
 ```
 
 Once socat is running it's possible to push images to the minikube registry from your local workstation:
 
-```
+```shell
 docker tag my/image localhost:5000/myimage
 docker push localhost:5000/myimage
 ```
diff --git a/site/content/en/docs/handbook/troubleshooting.md b/site/content/en/docs/handbook/troubleshooting.md
index bc44b458c747..ea627a695605 100644
--- a/site/content/en/docs/handbook/troubleshooting.md
+++ b/site/content/en/docs/handbook/troubleshooting.md
@@ -21,7 +21,9 @@ Example:
 
 minikube stores post-mortem INFO logs in the temporary directory of your system. On macOS or Linux, it's easy to get a list of recent INFO logs: 
 
-`find $TMPDIR -mtime -1 -type f -name "*minikube*INFO*" -ls  2>/dev/null`
+```shell
+find $TMPDIR -mtime -1 -type f -name "*minikube*INFO*" -ls  2>/dev/null
+```
 
 For instance, this shows:
 
@@ -29,7 +31,9 @@ For instance, this shows:
 
 These are plain text log files: you may rename them to "<filename>.log" and then drag/drop them into a GitHub issue for further analysis by the minikube team. You can quickly inspect the final lines of any of these logs via:
   
-`tail -n 10 <filename>`
+```shell
+tail -n 10 <filename>
+```
 
 for example, this shows:
 
@@ -83,4 +87,3 @@ minikube logs --problems
 ```
 
 This will attempt to surface known errors, such as invalid configuration flags. If nothing interesting shows up, try `minikube logs`.
-
diff --git a/site/content/en/docs/handbook/untrusted_certs.md b/site/content/en/docs/handbook/untrusted_certs.md
index 049f9075d77c..c17646ba7bb1 100644
--- a/site/content/en/docs/handbook/untrusted_certs.md
+++ b/site/content/en/docs/handbook/untrusted_certs.md
@@ -17,7 +17,7 @@ You may install the Root Certificate into the minikube cluster to access these c
 
 You will need a corporate X.509 Root Certificate in PEM format. If it's in DER format, convert it:
 
-```
+```shell
 openssl x509 -inform der -in my_company.cer -out my_company.pem
 ```
 
diff --git a/site/content/en/docs/start/_index.md b/site/content/en/docs/start/_index.md
index b15b010c7bca..2a4820727d03 100644
--- a/site/content/en/docs/start/_index.md
+++ b/site/content/en/docs/start/_index.md
@@ -58,7 +58,7 @@ brew install minikube
 
 If `which minikube` fails after installation via brew, you may have to remove the minikube cask and link the binary:
 
-```
+```shell
 brew cask remove minikube
 brew link minikube
 ```
@@ -160,7 +160,9 @@ minikube tunnel
 
 To find the routable IP, run this command and examine the `EXTERNAL-IP` column:
 
-`kubectl get services balanced`
+```shell
+kubectl get services balanced
+```
 
 Your deployment is now available at &lt;EXTERNAL-IP&gt;:8080
 
diff --git a/site/content/en/docs/tutorials/ambassador_ingress_controller.md b/site/content/en/docs/tutorials/ambassador_ingress_controller.md
index 30057a7e1013..7d6cb3693d73 100644
--- a/site/content/en/docs/tutorials/ambassador_ingress_controller.md
+++ b/site/content/en/docs/tutorials/ambassador_ingress_controller.md
@@ -84,7 +84,10 @@ spec:
           serviceName: hello-minikube
           servicePort: 8080
 ```
-Run the command: `kubectl apply -f hello-ingress.yaml`
+Run the command: 
+```shell
+kubectl apply -f hello-ingress.yaml
+```
 
 That's it! You can now access your service via Ambassador:
 ```shell script
@@ -120,7 +123,10 @@ spec:
   prefix: /hello-mapping/
   service: mapping-minikube.default:8080
 ```
-Run the command: `kubectl apply -f hello-mapping.yaml`
+Run the command: 
+```shell
+kubectl apply -f hello-mapping.yaml
+```
 
 That's it! You can now access your service via Ambassador:
 ```shell script
@@ -128,4 +134,4 @@ curl http://<Ambassdor's External IP'/hello-mapping/>
 ```
 
 **Note:** Read more about mappings in Ambassador's
-[documentation](https://www.getambassador.io/docs/latest/topics/using/mappings/).
\ No newline at end of file
+[documentation](https://www.getambassador.io/docs/latest/topics/using/mappings/).
diff --git a/site/content/en/docs/tutorials/multi_node.md b/site/content/en/docs/tutorials/multi_node.md
index 99a7e4e352d2..4872dc67abac 100644
--- a/site/content/en/docs/tutorials/multi_node.md
+++ b/site/content/en/docs/tutorials/multi_node.md
@@ -17,8 +17,10 @@ date: 2019-11-24
 ## Tutorial
 
 - Start a cluster with 2 nodes in the driver of your choice:
-```
+```shell
 minikube start --nodes 2 -p multinode-demo
+```
+```
 😄  [multinode-demo] minikube v1.10.1 on Darwin 10.15.4
 ✨  Automatically selected the hyperkit driver
 👍  Starting control plane node multinode-demo in cluster multinode-demo
@@ -40,16 +42,20 @@ To track progress on multi-node clusters, see https://github.com/kubernetes/mini
 ```
 
 - Get the list of your nodes:
-```
+```shell
 kubectl get nodes
+```
+```
 NAME                 STATUS   ROLES    AGE   VERSION
 multinode-demo       Ready    master   72s   v1.18.2
 multinode-demo-m02   Ready    <none>   33s   v1.18.2
 ```
 
 - You can also check the status of your nodes:
+```shell
+minikube status -p multinode-demo
+```
 ```
-$ minikube status -p multinode-demo
 multinode-demo
 type: Control Plane
 host: Running
@@ -64,33 +70,44 @@ kubelet: Running
 ```
 
 - Deploy our hello world deployment:
-```
+```shell
 kubectl apply -f hello-deployment.yaml
+```
+```
 deployment.apps/hello created
-
+```
+```shell
 kubectl rollout status deployment/hello
+```
+```
 deployment "hello" successfully rolled out
 ```
 
 
 - Deploy our hello world service, which just spits back the IP address the request was served from:
-```
+```shell
 kubectl apply -f hello-svc.yaml
+```
+```
 service/hello created
 ```
 
 
 - Check out the IP addresses of our pods, to note for future reference
-```
+```shell
 kubectl get pods -o wide
+```
+```
 NAME                    READY   STATUS    RESTARTS   AGE   IP           NODE             NOMINATED NODE   READINESS GATES
 hello-c7b8df44f-qbhxh   1/1     Running   0          31s   10.244.0.3   multinode-demo   <none>           <none>
 hello-c7b8df44f-xv4v6   1/1     Running   0          31s   10.244.0.2   multinode-demo   <none>           <none>
 ```
 
 - Look at our service, to know what URL to hit
-```
+```shell
 minikube service list -p multinode-demo
+```
+```
 |-------------|------------|--------------|-----------------------------|
 |  NAMESPACE  |    NAME    | TARGET PORT  |             URL             |
 |-------------|------------|--------------|-----------------------------|
@@ -101,8 +118,10 @@ minikube service list -p multinode-demo
 ```
 
 - Let's hit the URL a few times and see what comes back
-```
+```shell
 curl  http://192.168.64.226:31000
+```
+```
 Hello from hello-c7b8df44f-qbhxh (10.244.0.3)
 
 curl  http://192.168.64.226:31000
diff --git a/site/content/en/docs/tutorials/telemetry.md b/site/content/en/docs/tutorials/telemetry.md
index a41bafc520da..5c36284be2db 100644
--- a/site/content/en/docs/tutorials/telemetry.md
+++ b/site/content/en/docs/tutorials/telemetry.md
@@ -14,7 +14,7 @@ Currently, minikube supports the following exporters for tracing data:
 
 To collect trace data with minikube and the Stackdriver exporter, run:
 
-```
+```shell
 MINIKUBE_GCP_PROJECT_ID=<project ID> minikube start --output json --trace gcp
 ```
 
diff --git a/site/content/en/docs/tutorials/using_psp.md b/site/content/en/docs/tutorials/using_psp.md
index 38123c73c0af..55974c8a808d 100644
--- a/site/content/en/docs/tutorials/using_psp.md
+++ b/site/content/en/docs/tutorials/using_psp.md
@@ -20,7 +20,9 @@ This tutorial explains how to start minikube with Pod Security Policies (PSP) en
 Start minikube with the `PodSecurityPolicy` admission controller and the
 `pod-security-policy` addon enabled.
 
-`minikube start --extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy --addons=pod-security-policy`
+```shell
+minikube start --extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy --addons=pod-security-policy
+```
 
 The `pod-security-policy` addon must be enabled along with the admission
 controller to prevent issues during bootstrap.
@@ -35,12 +37,16 @@ the policies that addon enables must be separately applied to the cluster.
 Before starting minikube, you need to give it the PSP YAMLs in order to allow minikube to bootstrap.
 
 Create the directory:
-`mkdir -p ~/.minikube/files/etc/kubernetes/addons`
+```shell
+mkdir -p ~/.minikube/files/etc/kubernetes/addons
+```
 
 Copy the YAML below into this file: `~/.minikube/files/etc/kubernetes/addons/psp.yaml`
 
 Now start minikube:
-`minikube start --extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy`
+```shell
+minikube start --extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy
+```
 
 ```yaml
 ---
@@ -191,7 +197,7 @@ Next, apply the YAML shown above to the cluster.
 Finally, stop the cluster and then restart it with the admission controller
 enabled.
 
-```
+```shell
 minikube start
 kubectl apply -f /path/to/psp.yaml
 minikube stop

From 230c397aed5c772054686d4c5b255bb72527a2eb Mon Sep 17 00:00:00 2001
From: edtrist <edtrist@users.noreply.github.com>
Date: Tue, 1 Dec 2020 22:41:25 +0000
Subject: [PATCH 27/45] Custom GenMarkDown from Cobra to add shell language to
 runnable commands

---
 pkg/generate/docs.go | 97 ++++++++++++++++++++++++++------------------
 1 file changed, 58 insertions(+), 39 deletions(-)

diff --git a/pkg/generate/docs.go b/pkg/generate/docs.go
index dd8cb02fce83..63deb47414d0 100644
--- a/pkg/generate/docs.go
+++ b/pkg/generate/docs.go
@@ -17,19 +17,16 @@ limitations under the License.
 package generate
 
 import (
-	"bufio"
 	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
-	"github.com/spf13/cobra/doc"
 	"k8s.io/klog/v2"
 	"k8s.io/minikube/pkg/minikube/out"
 )
@@ -65,49 +62,71 @@ func DocForCommand(command *cobra.Command) (string, error) {
 	if err := writeSubcommands(command, buf); err != nil {
 		return "", errors.Wrap(err, "writing subcommands")
 	}
-	return removeHelpText(buf), nil
+	return buf.String(), nil
 }
 
-// after every command, cobra automatically appends
-// ### SEE ALSO
-
-// * [minikube addons](minikube_addons.md)	 - Modify minikube's Kubernetes addons
-
-// ###### Auto generated by spf13/cobra on 1-Apr-2020
-// help text which is unnecessary info after every subcommand
-// This function removes that text.
-func removeHelpText(buffer *bytes.Buffer) string {
-	beginningHelpText := "### SEE ALSO"
-	endHelpText := "###### Auto generated by spf13/cobra"
-	scanner := bufio.NewScanner(buffer)
-	includeLine := true
-
-	final := bytes.NewBuffer([]byte{})
-	for scanner.Scan() {
-		line := scanner.Text()
-		if strings.Contains(line, beginningHelpText) {
-			includeLine = false
-			continue
-		}
-		if strings.Contains(line, endHelpText) {
-			includeLine = true
-			continue
-		}
-		if !includeLine {
-			continue
-		}
-		// scanner strips the ending newline
-		if _, err := final.WriteString(line + "\n"); err != nil {
-			klog.Warningf("error removing help text: %v", err)
-			break
-		}
+// GenMarkdown creates markdown output.
+func GenMarkdown(cmd *cobra.Command, w io.Writer) error {
+	return GenMarkdownCustom(cmd, w, func(s string) string { return s })
+}
+
+// GenMarkdownCustom creates custom markdown output.
+func GenMarkdownCustom(cmd *cobra.Command, w io.Writer, linkHandler func(string) string) error {
+	cmd.InitDefaultHelpCmd()
+	cmd.InitDefaultHelpFlag()
+
+	buf := new(bytes.Buffer)
+	name := cmd.CommandPath()
+
+	short := cmd.Short
+	long := cmd.Long
+	if len(long) == 0 {
+		long = short
 	}
-	return final.String()
+
+	buf.WriteString("## " + name + "\n\n")
+	buf.WriteString(short + "\n\n")
+	buf.WriteString("### Synopsis\n\n")
+	buf.WriteString(long + "\n\n")
+
+	if cmd.Runnable() {
+		buf.WriteString(fmt.Sprintf("```shell\n%s\n```\n\n", cmd.UseLine()))
+	}
+
+	if len(cmd.Example) > 0 {
+		buf.WriteString("### Examples\n\n")
+		buf.WriteString(fmt.Sprintf("```\n%s\n```\n\n", cmd.Example))
+	}
+
+	if err := printOptions(buf, cmd, name); err != nil {
+		return err
+	}
+	_, err := buf.WriteTo(w)
+	return err
+}
+
+func printOptions(buf *bytes.Buffer, cmd *cobra.Command, name string) error {
+	flags := cmd.NonInheritedFlags()
+	flags.SetOutput(buf)
+	if flags.HasAvailableFlags() {
+		buf.WriteString("### Options\n\n```\n")
+		flags.PrintDefaults()
+		buf.WriteString("```\n\n")
+	}
+
+	parentFlags := cmd.InheritedFlags()
+	parentFlags.SetOutput(buf)
+	if parentFlags.HasAvailableFlags() {
+		buf.WriteString("### Options inherited from parent commands\n\n```\n")
+		parentFlags.PrintDefaults()
+		buf.WriteString("```\n\n")
+	}
+	return nil
 }
 
 // writeSubcommands recursively appends all subcommands to the doc
 func writeSubcommands(command *cobra.Command, w io.Writer) error {
-	if err := doc.GenMarkdown(command, w); err != nil {
+	if err := GenMarkdown(command, w); err != nil {
 		return errors.Wrapf(err, "getting markdown custom")
 	}
 	if !command.HasSubCommands() {

From 70b12370e78c96a48987d36f85854d0ab2b38260 Mon Sep 17 00:00:00 2001
From: edtrist <edtrist@users.noreply.github.com>
Date: Tue, 1 Dec 2020 22:42:03 +0000
Subject: [PATCH 28/45] Regenerate docs to add shell language to commands

---
 site/content/en/docs/commands/addons.md         | 14 +++++++-------
 site/content/en/docs/commands/cache.md          | 10 +++++-----
 site/content/en/docs/commands/completion.md     |  2 +-
 site/content/en/docs/commands/config.md         | 14 +++++++-------
 site/content/en/docs/commands/dashboard.md      |  2 +-
 site/content/en/docs/commands/delete.md         |  2 +-
 site/content/en/docs/commands/docker-env.md     |  2 +-
 site/content/en/docs/commands/help.md           |  2 +-
 site/content/en/docs/commands/ip.md             |  2 +-
 site/content/en/docs/commands/kubectl.md        |  2 +-
 site/content/en/docs/commands/logs.md           |  2 +-
 site/content/en/docs/commands/mount.md          |  2 +-
 site/content/en/docs/commands/node.md           | 14 +++++++-------
 site/content/en/docs/commands/pause.md          |  2 +-
 site/content/en/docs/commands/podman-env.md     |  2 +-
 site/content/en/docs/commands/profile.md        |  6 +++---
 site/content/en/docs/commands/service.md        |  6 +++---
 site/content/en/docs/commands/ssh-key.md        |  2 +-
 site/content/en/docs/commands/ssh.md            |  2 +-
 site/content/en/docs/commands/start.md          |  2 +-
 site/content/en/docs/commands/status.md         |  2 +-
 site/content/en/docs/commands/stop.md           |  2 +-
 site/content/en/docs/commands/tunnel.md         |  2 +-
 site/content/en/docs/commands/unpause.md        |  2 +-
 site/content/en/docs/commands/update-check.md   |  2 +-
 site/content/en/docs/commands/update-context.md |  2 +-
 site/content/en/docs/commands/version.md        |  2 +-
 27 files changed, 53 insertions(+), 53 deletions(-)

diff --git a/site/content/en/docs/commands/addons.md b/site/content/en/docs/commands/addons.md
index 621179f56feb..c558656d0dd5 100644
--- a/site/content/en/docs/commands/addons.md
+++ b/site/content/en/docs/commands/addons.md
@@ -13,7 +13,7 @@ Enable or disable a minikube addon
 
 addons modifies minikube addons files using subcommands like "minikube addons enable dashboard"
 
-```
+```shell
 minikube addons SUBCOMMAND [flags]
 ```
 
@@ -46,7 +46,7 @@ Configures the addon w/ADDON_NAME within minikube (example: minikube addons conf
 
 Configures the addon w/ADDON_NAME within minikube (example: minikube addons configure registry-creds). For a list of available addons use: minikube addons list
 
-```
+```shell
 minikube addons configure ADDON_NAME [flags]
 ```
 
@@ -79,7 +79,7 @@ Disables the addon w/ADDON_NAME within minikube (example: minikube addons disabl
 
 Disables the addon w/ADDON_NAME within minikube (example: minikube addons disable dashboard). For a list of available addons use: minikube addons list 
 
-```
+```shell
 minikube addons disable ADDON_NAME [flags]
 ```
 
@@ -112,7 +112,7 @@ Enables the addon w/ADDON_NAME within minikube (example: minikube addons enable
 
 Enables the addon w/ADDON_NAME within minikube (example: minikube addons enable dashboard). For a list of available addons use: minikube addons list 
 
-```
+```shell
 minikube addons enable ADDON_NAME [flags]
 ```
 
@@ -146,7 +146,7 @@ Help about any command
 Help provides help for any command in the application.
 Simply type addons help [path to command] for full details.
 
-```
+```shell
 minikube addons help [command] [flags]
 ```
 
@@ -179,7 +179,7 @@ Lists all available minikube addons as well as their current statuses (enabled/d
 
 Lists all available minikube addons as well as their current statuses (enabled/disabled)
 
-```
+```shell
 minikube addons list [flags]
 ```
 
@@ -218,7 +218,7 @@ Opens the addon w/ADDON_NAME within minikube (example: minikube addons open dash
 
 Opens the addon w/ADDON_NAME within minikube (example: minikube addons open dashboard). For a list of available addons use: minikube addons list 
 
-```
+```shell
 minikube addons open ADDON_NAME [flags]
 ```
 
diff --git a/site/content/en/docs/commands/cache.md b/site/content/en/docs/commands/cache.md
index 3917b19b701f..c5694102cb60 100644
--- a/site/content/en/docs/commands/cache.md
+++ b/site/content/en/docs/commands/cache.md
@@ -42,7 +42,7 @@ Add an image to local cache.
 
 Add an image to local cache.
 
-```
+```shell
 minikube cache add [flags]
 ```
 
@@ -75,7 +75,7 @@ Delete an image from the local cache.
 
 Delete an image from the local cache.
 
-```
+```shell
 minikube cache delete [flags]
 ```
 
@@ -109,7 +109,7 @@ Help about any command
 Help provides help for any command in the application.
 Simply type cache help [path to command] for full details.
 
-```
+```shell
 minikube cache help [command] [flags]
 ```
 
@@ -142,7 +142,7 @@ List all available images from the local cache.
 
 List all available images from the local cache.
 
-```
+```shell
 minikube cache list [flags]
 ```
 
@@ -182,7 +182,7 @@ reload cached images.
 
 reloads images previously added using the 'cache add' subcommand
 
-```
+```shell
 minikube cache reload [flags]
 ```
 
diff --git a/site/content/en/docs/commands/completion.md b/site/content/en/docs/commands/completion.md
index 7738a2e2161d..0b10bf41d9b6 100644
--- a/site/content/en/docs/commands/completion.md
+++ b/site/content/en/docs/commands/completion.md
@@ -34,7 +34,7 @@ Outputs minikube shell completion for the given shell (bash, zsh or fish)
 	Note for fish users: [2] please refer to this docs for more details https://fishshell.com/docs/current/#tab-completion
 
 
-```
+```shell
 minikube completion SHELL [flags]
 ```
 
diff --git a/site/content/en/docs/commands/config.md b/site/content/en/docs/commands/config.md
index 8e66d3f1938b..40c9c9571f35 100644
--- a/site/content/en/docs/commands/config.md
+++ b/site/content/en/docs/commands/config.md
@@ -43,7 +43,7 @@ Configurable fields:
  * embed-certs
  * native-ssh
 
-```
+```shell
 minikube config SUBCOMMAND [flags]
 ```
 
@@ -79,7 +79,7 @@ Acceptable fields:
 
  * driver
 
-```
+```shell
 minikube config defaults PROPERTY_NAME [flags]
 ```
 
@@ -118,7 +118,7 @@ Gets the value of PROPERTY_NAME from the minikube config file
 
 Returns the value of PROPERTY_NAME from the minikube config file.  Can be overwritten at runtime by flags or environmental variables.
 
-```
+```shell
 minikube config get PROPERTY_NAME [flags]
 ```
 
@@ -152,7 +152,7 @@ Help about any command
 Help provides help for any command in the application.
 Simply type config help [path to command] for full details.
 
-```
+```shell
 minikube config help [command] [flags]
 ```
 
@@ -186,7 +186,7 @@ Sets an individual value in a minikube config file
 Sets the PROPERTY_NAME config value to PROPERTY_VALUE
 	These values can be overwritten by flags or environment variables at runtime.
 
-```
+```shell
 minikube config set PROPERTY_NAME PROPERTY_VALUE [flags]
 ```
 
@@ -219,7 +219,7 @@ unsets an individual value in a minikube config file
 
 unsets PROPERTY_NAME from the minikube config file.  Can be overwritten by flags or environmental variables
 
-```
+```shell
 minikube config unset PROPERTY_NAME [flags]
 ```
 
@@ -252,7 +252,7 @@ Display values currently set in the minikube config file
 
 Display values currently set in the minikube config file.
 
-```
+```shell
 minikube config view [flags]
 ```
 
diff --git a/site/content/en/docs/commands/dashboard.md b/site/content/en/docs/commands/dashboard.md
index 8df23e2c9364..a4cba95564e2 100644
--- a/site/content/en/docs/commands/dashboard.md
+++ b/site/content/en/docs/commands/dashboard.md
@@ -13,7 +13,7 @@ Access the Kubernetes dashboard running within the minikube cluster
 
 Access the Kubernetes dashboard running within the minikube cluster
 
-```
+```shell
 minikube dashboard [flags]
 ```
 
diff --git a/site/content/en/docs/commands/delete.md b/site/content/en/docs/commands/delete.md
index 34cea1451810..65310fdea5b1 100644
--- a/site/content/en/docs/commands/delete.md
+++ b/site/content/en/docs/commands/delete.md
@@ -14,7 +14,7 @@ Deletes a local Kubernetes cluster
 Deletes a local Kubernetes cluster. This command deletes the VM, and removes all
 associated files.
 
-```
+```shell
 minikube delete [flags]
 ```
 
diff --git a/site/content/en/docs/commands/docker-env.md b/site/content/en/docs/commands/docker-env.md
index 7ce914a06309..39b939d88082 100644
--- a/site/content/en/docs/commands/docker-env.md
+++ b/site/content/en/docs/commands/docker-env.md
@@ -13,7 +13,7 @@ Configure environment to use minikube's Docker daemon
 
 Sets up docker env variables; similar to '$(docker-machine env)'.
 
-```
+```shell
 minikube docker-env [flags]
 ```
 
diff --git a/site/content/en/docs/commands/help.md b/site/content/en/docs/commands/help.md
index 6f25e68ca812..3bc45fe4e96c 100644
--- a/site/content/en/docs/commands/help.md
+++ b/site/content/en/docs/commands/help.md
@@ -14,7 +14,7 @@ Help about any command
 Help provides help for any command in the application.
 Simply type minikube help [path to command] for full details.
 
-```
+```shell
 minikube help [command] [flags]
 ```
 
diff --git a/site/content/en/docs/commands/ip.md b/site/content/en/docs/commands/ip.md
index 0a14345642f4..044e4435d7ac 100644
--- a/site/content/en/docs/commands/ip.md
+++ b/site/content/en/docs/commands/ip.md
@@ -13,7 +13,7 @@ Retrieves the IP address of the running cluster
 
 Retrieves the IP address of the running cluster, and writes it to STDOUT.
 
-```
+```shell
 minikube ip [flags]
 ```
 
diff --git a/site/content/en/docs/commands/kubectl.md b/site/content/en/docs/commands/kubectl.md
index a787c52b5bbb..f3e231b35c35 100644
--- a/site/content/en/docs/commands/kubectl.md
+++ b/site/content/en/docs/commands/kubectl.md
@@ -17,7 +17,7 @@ Examples:
 minikube kubectl -- --help
 minikube kubectl -- get pods --namespace kube-system
 
-```
+```shell
 minikube kubectl [flags]
 ```
 
diff --git a/site/content/en/docs/commands/logs.md b/site/content/en/docs/commands/logs.md
index 1734f8b7b988..f84208137c1d 100644
--- a/site/content/en/docs/commands/logs.md
+++ b/site/content/en/docs/commands/logs.md
@@ -13,7 +13,7 @@ Returns logs to debug a local Kubernetes cluster
 
 Gets the logs of the running instance, used for debugging minikube, not user code.
 
-```
+```shell
 minikube logs [flags]
 ```
 
diff --git a/site/content/en/docs/commands/mount.md b/site/content/en/docs/commands/mount.md
index 8d9decd5909c..3d6409c178a3 100644
--- a/site/content/en/docs/commands/mount.md
+++ b/site/content/en/docs/commands/mount.md
@@ -13,7 +13,7 @@ Mounts the specified directory into minikube
 
 Mounts the specified directory into minikube.
 
-```
+```shell
 minikube mount [flags] <source directory>:<target directory>
 ```
 
diff --git a/site/content/en/docs/commands/node.md b/site/content/en/docs/commands/node.md
index e7c22ec26cb0..79778a7dfa32 100644
--- a/site/content/en/docs/commands/node.md
+++ b/site/content/en/docs/commands/node.md
@@ -13,7 +13,7 @@ Add, remove, or list additional nodes
 
 Operations on nodes
 
-```
+```shell
 minikube node [flags]
 ```
 
@@ -46,7 +46,7 @@ Adds a node to the given cluster.
 
 Adds a node to the given cluster config, and starts it.
 
-```
+```shell
 minikube node add [flags]
 ```
 
@@ -87,7 +87,7 @@ Deletes a node from a cluster.
 
 Deletes a node from a cluster.
 
-```
+```shell
 minikube node delete [flags]
 ```
 
@@ -121,7 +121,7 @@ Help about any command
 Help provides help for any command in the application.
 Simply type node help [path to command] for full details.
 
-```
+```shell
 minikube node help [command] [flags]
 ```
 
@@ -154,7 +154,7 @@ List nodes.
 
 List existing minikube nodes.
 
-```
+```shell
 minikube node list [flags]
 ```
 
@@ -187,7 +187,7 @@ Starts a node.
 
 Starts an existing stopped node in a cluster.
 
-```
+```shell
 minikube node start [flags]
 ```
 
@@ -226,7 +226,7 @@ Stops a node in a cluster.
 
 Stops a node in a cluster.
 
-```
+```shell
 minikube node stop [flags]
 ```
 
diff --git a/site/content/en/docs/commands/pause.md b/site/content/en/docs/commands/pause.md
index 26938f43a214..b6463f1ff9b5 100644
--- a/site/content/en/docs/commands/pause.md
+++ b/site/content/en/docs/commands/pause.md
@@ -13,7 +13,7 @@ pause Kubernetes
 
 pause Kubernetes
 
-```
+```shell
 minikube pause [flags]
 ```
 
diff --git a/site/content/en/docs/commands/podman-env.md b/site/content/en/docs/commands/podman-env.md
index ef94b796b319..4ea853f8e1d9 100644
--- a/site/content/en/docs/commands/podman-env.md
+++ b/site/content/en/docs/commands/podman-env.md
@@ -13,7 +13,7 @@ Configure environment to use minikube's Podman service
 
 Sets up podman env variables; similar to '$(podman-machine env)'.
 
-```
+```shell
 minikube podman-env [flags]
 ```
 
diff --git a/site/content/en/docs/commands/profile.md b/site/content/en/docs/commands/profile.md
index eb1ea697a33f..9f26ea434f11 100644
--- a/site/content/en/docs/commands/profile.md
+++ b/site/content/en/docs/commands/profile.md
@@ -13,7 +13,7 @@ Get or list the current profiles (clusters)
 
 profile sets the current minikube profile, or gets the current profile if no arguments are provided.  This is used to run and manage multiple minikube instance.  You can return to the default minikube profile by running `minikube profile default`
 
-```
+```shell
 minikube profile [MINIKUBE_PROFILE_NAME].  You can return to the default minikube profile by running `minikube profile default` [flags]
 ```
 
@@ -47,7 +47,7 @@ Help about any command
 Help provides help for any command in the application.
 Simply type profile help [path to command] for full details.
 
-```
+```shell
 minikube profile help [command] [flags]
 ```
 
@@ -80,7 +80,7 @@ Lists all minikube profiles.
 
 Lists all valid minikube profiles and detects all possible invalid profiles.
 
-```
+```shell
 minikube profile list [flags]
 ```
 
diff --git a/site/content/en/docs/commands/service.md b/site/content/en/docs/commands/service.md
index c387e9f63719..e10846d67009 100644
--- a/site/content/en/docs/commands/service.md
+++ b/site/content/en/docs/commands/service.md
@@ -13,7 +13,7 @@ Returns a URL to connect to a service
 
 Returns the Kubernetes URL for a service in your local cluster. In the case of multiple URLs they will be printed one at a time.
 
-```
+```shell
 minikube service [flags] SERVICE
 ```
 
@@ -58,7 +58,7 @@ Help about any command
 Help provides help for any command in the application.
 Simply type service help [path to command] for full details.
 
-```
+```shell
 minikube service help [command] [flags]
 ```
 
@@ -92,7 +92,7 @@ Lists the URLs for the services in your local cluster
 
 Lists the URLs for the services in your local cluster
 
-```
+```shell
 minikube service list [flags]
 ```
 
diff --git a/site/content/en/docs/commands/ssh-key.md b/site/content/en/docs/commands/ssh-key.md
index 89d84187f01a..5abba114ad0d 100644
--- a/site/content/en/docs/commands/ssh-key.md
+++ b/site/content/en/docs/commands/ssh-key.md
@@ -13,7 +13,7 @@ Retrieve the ssh identity key path of the specified cluster
 
 Retrieve the ssh identity key path of the specified cluster.
 
-```
+```shell
 minikube ssh-key [flags]
 ```
 
diff --git a/site/content/en/docs/commands/ssh.md b/site/content/en/docs/commands/ssh.md
index 3a00f98d1eb9..bf37dedc52a1 100644
--- a/site/content/en/docs/commands/ssh.md
+++ b/site/content/en/docs/commands/ssh.md
@@ -13,7 +13,7 @@ Log into the minikube environment (for debugging)
 
 Log into or run a command on a machine with SSH; similar to 'docker-machine ssh'.
 
-```
+```shell
 minikube ssh [flags]
 ```
 
diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index 52715a96b276..d24e0f719ed7 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -13,7 +13,7 @@ Starts a local Kubernetes cluster
 
 Starts a local Kubernetes cluster
 
-```
+```shell
 minikube start [flags]
 ```
 
diff --git a/site/content/en/docs/commands/status.md b/site/content/en/docs/commands/status.md
index 07eb6da74596..f4902dab0baf 100644
--- a/site/content/en/docs/commands/status.md
+++ b/site/content/en/docs/commands/status.md
@@ -15,7 +15,7 @@ Gets the status of a local Kubernetes cluster.
 	Exit status contains the status of minikube's VM, cluster and Kubernetes encoded on it's bits in this order from right to left.
 	Eg: 7 meaning: 1 (for minikube NOK) + 2 (for cluster NOK) + 4 (for Kubernetes NOK)
 
-```
+```shell
 minikube status [flags]
 ```
 
diff --git a/site/content/en/docs/commands/stop.md b/site/content/en/docs/commands/stop.md
index 8b96e3de894c..34db14a875f4 100644
--- a/site/content/en/docs/commands/stop.md
+++ b/site/content/en/docs/commands/stop.md
@@ -13,7 +13,7 @@ Stops a running local Kubernetes cluster
 
 Stops a local Kubernetes cluster. This command stops the underlying VM or container, but keeps user data intact. The cluster can be started again with the "start" command.
 
-```
+```shell
 minikube stop [flags]
 ```
 
diff --git a/site/content/en/docs/commands/tunnel.md b/site/content/en/docs/commands/tunnel.md
index 370e8128dcc1..62cc1510265c 100644
--- a/site/content/en/docs/commands/tunnel.md
+++ b/site/content/en/docs/commands/tunnel.md
@@ -13,7 +13,7 @@ Connect to LoadBalancer services
 
 tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP. for a detailed example see https://minikube.sigs.k8s.io/docs/tasks/loadbalancer
 
-```
+```shell
 minikube tunnel [flags]
 ```
 
diff --git a/site/content/en/docs/commands/unpause.md b/site/content/en/docs/commands/unpause.md
index aabcedf80b74..5ee5c52a7be4 100644
--- a/site/content/en/docs/commands/unpause.md
+++ b/site/content/en/docs/commands/unpause.md
@@ -13,7 +13,7 @@ unpause Kubernetes
 
 unpause Kubernetes
 
-```
+```shell
 minikube unpause [flags]
 ```
 
diff --git a/site/content/en/docs/commands/update-check.md b/site/content/en/docs/commands/update-check.md
index b3d98bb1bb67..f4a66c76022f 100644
--- a/site/content/en/docs/commands/update-check.md
+++ b/site/content/en/docs/commands/update-check.md
@@ -13,7 +13,7 @@ Print current and latest version number
 
 Print current and latest version number
 
-```
+```shell
 minikube update-check [flags]
 ```
 
diff --git a/site/content/en/docs/commands/update-context.md b/site/content/en/docs/commands/update-context.md
index b981dbeba55a..2725cb81d289 100644
--- a/site/content/en/docs/commands/update-context.md
+++ b/site/content/en/docs/commands/update-context.md
@@ -14,7 +14,7 @@ Update kubeconfig in case of an IP or port change
 Retrieves the IP address of the running cluster, checks it
 			with IP in kubeconfig, and corrects kubeconfig if incorrect.
 
-```
+```shell
 minikube update-context [flags]
 ```
 
diff --git a/site/content/en/docs/commands/version.md b/site/content/en/docs/commands/version.md
index d1fc7e06aa90..fc223a2e77f5 100644
--- a/site/content/en/docs/commands/version.md
+++ b/site/content/en/docs/commands/version.md
@@ -13,7 +13,7 @@ Print the version of minikube
 
 Print the version of minikube.
 
-```
+```shell
 minikube version [flags]
 ```
 

From 1440438020d73dc317cd092257543e1847c0ae8b Mon Sep 17 00:00:00 2001
From: edtrist <edtrist@users.noreply.github.com>
Date: Tue, 1 Dec 2020 23:25:38 +0000
Subject: [PATCH 29/45] Remove unused argument

---
 pkg/generate/docs.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/generate/docs.go b/pkg/generate/docs.go
index 63deb47414d0..fefc720d8ee0 100644
--- a/pkg/generate/docs.go
+++ b/pkg/generate/docs.go
@@ -98,14 +98,14 @@ func GenMarkdownCustom(cmd *cobra.Command, w io.Writer, linkHandler func(string)
 		buf.WriteString(fmt.Sprintf("```\n%s\n```\n\n", cmd.Example))
 	}
 
-	if err := printOptions(buf, cmd, name); err != nil {
+	if err := printOptions(buf, cmd); err != nil {
 		return err
 	}
 	_, err := buf.WriteTo(w)
 	return err
 }
 
-func printOptions(buf *bytes.Buffer, cmd *cobra.Command, name string) error {
+func printOptions(buf *bytes.Buffer, cmd *cobra.Command) error {
 	flags := cmd.NonInheritedFlags()
 	flags.SetOutput(buf)
 	if flags.HasAvailableFlags() {

From 5553f48e3617505bd219aa241d7ace83ee0321dc Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 1 Dec 2020 19:37:15 -0800
Subject: [PATCH 30/45] Add --label flag so we can add custom labels to metric
 collection

We can later use this flag to specify environment (Cloud Shell, OS), or driver used.
---
 Makefile                |  5 +++++
 hack/metrics/metrics.go | 35 +++++++++++++++++++++++++++--------
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/Makefile b/Makefile
index 836d7f78b89c..52893512be24 100644
--- a/Makefile
+++ b/Makefile
@@ -767,6 +767,11 @@ out/mkcmp:
 out/performance-bot:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/pr-bot/bot.go
 
+.PHONY: out/metrics-collector
+out/metrics-collector:
+	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ hack/metrics/*.go
+
+
 .PHONY: compare
 compare: out/mkcmp out/minikube
 	mv out/minikube out/$(CURRENT_GIT_BRANCH).minikube
diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go
index cd0c2ded935f..9a846b6c1c00 100644
--- a/hack/metrics/metrics.go
+++ b/hack/metrics/metrics.go
@@ -18,11 +18,13 @@ package main
 
 import (
 	"context"
+	"flag"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
+	"strings"
 	"time"
 
 	_ "cloud.google.com/go/storage"
@@ -30,7 +32,6 @@ import (
 	"github.com/pkg/errors"
 	"go.opencensus.io/stats"
 	"go.opencensus.io/stats/view"
-	"go.opencensus.io/tag"
 	"go.opencensus.io/trace"
 	pkgtrace "k8s.io/minikube/pkg/trace"
 )
@@ -43,6 +44,7 @@ const (
 var (
 	// The task latency in seconds
 	latencyS = stats.Float64("repl/start_time", "start time in seconds", "s")
+	labels   string
 )
 
 func main() {
@@ -52,17 +54,17 @@ func main() {
 	}
 }
 
+func init() {
+	flag.StringVar(&labels, "label", "", "Comma separated list of labels to add to metrics in key:value format")
+	flag.Parse()
+}
+
 func execute() error {
 	projectID := os.Getenv(pkgtrace.ProjectEnvVar)
 	if projectID == "" {
 		return fmt.Errorf("metrics collector requires a valid GCP project id set via the %s env variable", pkgtrace.ProjectEnvVar)
 	}
 
-	osMethod, err := tag.NewKey("os")
-	if err != nil {
-		return errors.Wrap(err, "new tag key")
-	}
-
 	// Register the view. It is imperative that this step exists,
 	// otherwise recorded metrics will be dropped and never exported.
 	v := &view.View{
@@ -70,7 +72,6 @@ func execute() error {
 		Measure:     latencyS,
 		Description: "minikube start time",
 		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{osMethod},
 	}
 	if err := view.Register(v); err != nil {
 		return errors.Wrap(err, "registering view")
@@ -82,7 +83,8 @@ func execute() error {
 		MetricPrefix: "minikube_start",
 		// ReportingInterval sets the frequency of reporting metrics
 		// to stackdriver backend.
-		ReportingInterval: 1 * time.Second,
+		ReportingInterval:       1 * time.Second,
+		DefaultMonitoringLabels: getLabels(),
 	})
 	if err != nil {
 		return errors.Wrap(err, "getting stackdriver exporter")
@@ -122,6 +124,23 @@ func execute() error {
 	}
 }
 
+func getLabels() *stackdriver.Labels {
+	l := &stackdriver.Labels{}
+	if labels == "" {
+		return l
+	}
+	separated := strings.Split(labels, ",")
+	for _, s := range separated {
+		sep := strings.Split(s, ":")
+		if len(sep) != 2 {
+			continue
+		}
+		log.Printf("Adding label %s=%s to metrics...", sep[0], sep[1])
+		l.Set(sep[0], sep[1], "")
+	}
+	return l
+}
+
 func minikubeStartTime(ctx context.Context, projectID, minikubePath string) (float64, error) {
 	if err := downloadMinikube(ctx, minikubePath); err != nil {
 		return 0, errors.Wrap(err, "downloading minikube")

From 89bb352c6de39ceb13c9d2b8010c7e48ce2b6ef5 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Wed, 2 Dec 2020 11:02:27 -0800
Subject: [PATCH 31/45] specify temp file via flag and don't run infintely

---
 hack/metrics/metrics.go | 29 +++++++++--------------------
 1 file changed, 9 insertions(+), 20 deletions(-)

diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go
index 9a846b6c1c00..013a900d1633 100644
--- a/hack/metrics/metrics.go
+++ b/hack/metrics/metrics.go
@@ -20,7 +20,6 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -45,6 +44,7 @@ var (
 	// The task latency in seconds
 	latencyS = stats.Float64("repl/start_time", "start time in seconds", "s")
 	labels   string
+	tmpFile  string
 )
 
 func main() {
@@ -56,6 +56,7 @@ func main() {
 
 func init() {
 	flag.StringVar(&labels, "label", "", "Comma separated list of labels to add to metrics in key:value format")
+	flag.StringVar(&tmpFile, "file", "/tmp/minikube", "Download minikube to this file for testing")
 	flag.Parse()
 }
 
@@ -101,27 +102,15 @@ func execute() error {
 	defer sd.StopMetricsExporter()
 	ctx := context.Background()
 
-	// create a tmp file to download minikube to
-	tmp, err := ioutil.TempFile("", binary())
+	// track minikube start time and record it to metrics collector
+	st, err := minikubeStartTime(ctx, projectID, tmpFile)
 	if err != nil {
-		return errors.Wrap(err, "creating tmp file")
-	}
-	if err := tmp.Close(); err != nil {
-		return errors.Wrap(err, "closing file")
-	}
-	defer os.Remove(tmp.Name())
-
-	// start loop where we will track minikube start time
-	for {
-		st, err := minikubeStartTime(ctx, projectID, tmp.Name())
-		if err != nil {
-			log.Printf("error collecting start time: %v", err)
-			continue
-		}
-		fmt.Printf("Latency: %f\n", st)
-		stats.Record(ctx, latencyS.M(st))
-		time.Sleep(30 * time.Second)
+		return errors.Wrap(err, "collecting start time")
 	}
+	fmt.Printf("Latency: %f\n", st)
+	stats.Record(ctx, latencyS.M(st))
+	time.Sleep(30 * time.Second)
+	return nil
 }
 
 func getLabels() *stackdriver.Labels {

From 8916ac46ea2e5080cd0174c7e9b91d8574e0b6f1 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Wed, 2 Dec 2020 13:08:54 -0800
Subject: [PATCH 32/45] update storage provsioner

---
 go.mod                                           |  1 -
 go.sum                                           |  7 +++++--
 pkg/minikube/bootstrapper/images/images_test.go  |  4 ++--
 pkg/minikube/bootstrapper/images/kubeadm_test.go | 12 ++++++------
 pkg/minikube/cruntime/containerd_test.go         |  2 +-
 pkg/minikube/download/preload.go                 |  2 +-
 6 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/go.mod b/go.mod
index b3c9278dee72..a3d449a00a09 100644
--- a/go.mod
+++ b/go.mod
@@ -75,7 +75,6 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f
 	github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
-	go.opencensus.io v0.22.4
 	go.opentelemetry.io/otel v0.13.0
 	go.opentelemetry.io/otel/sdk v0.13.0
 	golang.org/x/build v0.0.0-20190927031335-2835ba2e683f
diff --git a/go.sum b/go.sum
index a0046dc65280..54b82a0fb782 100644
--- a/go.sum
+++ b/go.sum
@@ -93,7 +93,6 @@ github.com/Djarvur/go-err113 v0.0.0-20200410182137-af658d038157/go.mod h1:4UJr5H
 github.com/Djarvur/go-err113 v0.1.0/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
 github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20191009163259-e802c2cb94ae/go.mod h1:mjwGPas4yKduTyubHvD1Atl9r1rUq8DfVy+gkVvZ+oo=
 github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14=
-github.com/GoogleCloudPlatform/opentelemetry-operations-go v0.13.0 h1:Gx9IxOjR9ug5Ad8YbafxJ6N2g6oDj/Q419tYHdd1od4=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.13.0 h1:fjKUtfldCPIF4nIzAAj3LzP8Lrd3DuRIMiFdOsj4fLc=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.13.0/go.mod h1:q/paYxLXKVhwfC3lzLfhtL54fAx14wzMN9DundQOBMc=
 github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA=
@@ -555,6 +554,7 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE=
 github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f h1:Jnx61latede7zDD3DiiP4gmNz33uK0U5HDUaF0a/HVQ=
@@ -585,6 +585,7 @@ github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTV
 github.com/googleapis/gnostic v0.2.2/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
 github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleinterns/cloud-operations-api-mock v0.0.0-20200709193332-a1e58c29bdd3 h1:eHv/jVY/JNop1xg2J9cBb4EzyMpWZoNCP1BslSAIkOI=
 github.com/googleinterns/cloud-operations-api-mock v0.0.0-20200709193332-a1e58c29bdd3/go.mod h1:h/KNeRx7oYU4SpA4SoY7W2/NxDKEEVuwA6j9A27L4OI=
 github.com/gookit/color v1.2.4/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
@@ -1105,6 +1106,7 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
@@ -1190,7 +1192,6 @@ go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opentelemetry.io v0.1.0 h1:EANZoRCOP+A3faIlw/iN6YEWoYb1vleZRKm1EvH8T48=
 go.opentelemetry.io/otel v0.13.0 h1:2isEnyzjjJZq6r2EKMsFj4TxiQiexsM04AVhwbR/oBA=
 go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY=
 go.opentelemetry.io/otel/sdk v0.13.0 h1:4VCfpKamZ8GtnepXxMRurSpHpMKkcxhtO33z1S4rGDQ=
@@ -1503,6 +1504,7 @@ golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d h1:SR+e35rACZFBohNb4Om1ibX
 golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200701151220-7cb253f4c4f8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200713011307-fd294ab11aed h1:+qzWo37K31KxduIYaBeMqJ8MUOyTayOQKpH9aDPLMSY=
 golang.org/x/tools v0.0.0-20200713011307-fd294ab11aed/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
@@ -1661,6 +1663,7 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
diff --git a/pkg/minikube/bootstrapper/images/images_test.go b/pkg/minikube/bootstrapper/images/images_test.go
index fd3743a8a8dc..5831e450c699 100644
--- a/pkg/minikube/bootstrapper/images/images_test.go
+++ b/pkg/minikube/bootstrapper/images/images_test.go
@@ -24,7 +24,7 @@ import (
 
 func TestAuxiliary(t *testing.T) {
 	want := []string{
-		"gcr.io/k8s-minikube/storage-provisioner:v3",
+		"gcr.io/k8s-minikube/storage-provisioner:v4",
 		"docker.io/kubernetesui/dashboard:v2.0.3",
 		"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 	}
@@ -36,7 +36,7 @@ func TestAuxiliary(t *testing.T) {
 
 func TestAuxiliaryMirror(t *testing.T) {
 	want := []string{
-		"test.mirror/storage-provisioner:v3",
+		"test.mirror/storage-provisioner:v4",
 		"test.mirror/dashboard:v2.0.3",
 		"test.mirror/metrics-scraper:v1.0.4",
 	}
diff --git a/pkg/minikube/bootstrapper/images/kubeadm_test.go b/pkg/minikube/bootstrapper/images/kubeadm_test.go
index 36f976d9e7ae..81d37bf8ed46 100644
--- a/pkg/minikube/bootstrapper/images/kubeadm_test.go
+++ b/pkg/minikube/bootstrapper/images/kubeadm_test.go
@@ -37,7 +37,7 @@ func TestKubeadmImages(t *testing.T) {
 			"k8s.gcr.io/coredns:1.6.5",
 			"k8s.gcr.io/etcd:3.4.3-0",
 			"k8s.gcr.io/pause:3.1",
-			"gcr.io/k8s-minikube/storage-provisioner:v3",
+			"gcr.io/k8s-minikube/storage-provisioner:v4",
 			"docker.io/kubernetesui/dashboard:v2.0.3",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
@@ -49,7 +49,7 @@ func TestKubeadmImages(t *testing.T) {
 			"mirror.k8s.io/coredns:1.6.2",
 			"mirror.k8s.io/etcd:3.3.15-0",
 			"mirror.k8s.io/pause:3.1",
-			"mirror.k8s.io/storage-provisioner:v3",
+			"mirror.k8s.io/storage-provisioner:v4",
 			"mirror.k8s.io/dashboard:v2.0.3",
 			"mirror.k8s.io/metrics-scraper:v1.0.4",
 		}},
@@ -61,7 +61,7 @@ func TestKubeadmImages(t *testing.T) {
 			"k8s.gcr.io/coredns:1.3.1",
 			"k8s.gcr.io/etcd:3.3.10",
 			"k8s.gcr.io/pause:3.1",
-			"gcr.io/k8s-minikube/storage-provisioner:v3",
+			"gcr.io/k8s-minikube/storage-provisioner:v4",
 			"docker.io/kubernetesui/dashboard:v2.0.3",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
@@ -73,7 +73,7 @@ func TestKubeadmImages(t *testing.T) {
 			"k8s.gcr.io/coredns:1.3.1",
 			"k8s.gcr.io/etcd:3.3.10",
 			"k8s.gcr.io/pause:3.1",
-			"gcr.io/k8s-minikube/storage-provisioner:v3",
+			"gcr.io/k8s-minikube/storage-provisioner:v4",
 			"docker.io/kubernetesui/dashboard:v2.0.3",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
@@ -85,7 +85,7 @@ func TestKubeadmImages(t *testing.T) {
 			"k8s.gcr.io/coredns:1.2.6",
 			"k8s.gcr.io/etcd:3.2.24",
 			"k8s.gcr.io/pause:3.1",
-			"gcr.io/k8s-minikube/storage-provisioner:v3",
+			"gcr.io/k8s-minikube/storage-provisioner:v4",
 			"docker.io/kubernetesui/dashboard:v2.0.3",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
@@ -97,7 +97,7 @@ func TestKubeadmImages(t *testing.T) {
 			"k8s.gcr.io/coredns:1.2.2",
 			"k8s.gcr.io/etcd:3.2.24",
 			"k8s.gcr.io/pause:3.1",
-			"gcr.io/k8s-minikube/storage-provisioner:v3",
+			"gcr.io/k8s-minikube/storage-provisioner:v4",
 			"docker.io/kubernetesui/dashboard:v2.0.3",
 			"docker.io/kubernetesui/metrics-scraper:v1.0.4",
 		}},
diff --git a/pkg/minikube/cruntime/containerd_test.go b/pkg/minikube/cruntime/containerd_test.go
index 7fb4846f3849..bd7227fc7752 100644
--- a/pkg/minikube/cruntime/containerd_test.go
+++ b/pkg/minikube/cruntime/containerd_test.go
@@ -27,7 +27,7 @@ func TestAddRepoTagToImageName(t *testing.T) {
 	}{
 		{"kubernetesui/dashboard:v2.0.3", "docker.io/kubernetesui/dashboard:v2.0.3"},
 		{"kubernetesui/metrics-scraper:v1.0.4", "docker.io/kubernetesui/metrics-scraper:v1.0.4"},
-		{"gcr.io/k8s-minikube/storage-provisioner:v3", "gcr.io/k8s-minikube/storage-provisioner:v3"},
+		{"gcr.io/k8s-minikube/storage-provisioner:v4", "gcr.io/k8s-minikube/storage-provisioner:v4"},
 	}
 	for _, tc := range tests {
 		t.Run(tc.imgName, func(t *testing.T) {
diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go
index 668c4edf56fc..713495b327bd 100644
--- a/pkg/minikube/download/preload.go
+++ b/pkg/minikube/download/preload.go
@@ -41,7 +41,7 @@ const (
 	// PreloadVersion is the current version of the preloaded tarball
 	//
 	// NOTE: You may need to bump this version up when upgrading auxiliary docker images
-	PreloadVersion = "v6"
+	PreloadVersion = "v7"
 	// PreloadBucket is the name of the GCS bucket where preloaded volume tarballs exist
 	PreloadBucket = "minikube-preloaded-volume-tarballs"
 )

From 8c2e03b2a722b82ed4b4e39c821b0369def854b6 Mon Sep 17 00:00:00 2001
From: Ling Samuel <lingsamuelgrace@gmail.com>
Date: Thu, 3 Dec 2020 11:22:44 +0800
Subject: [PATCH 33/45] Fix missing InitialSetup in `node start`

Signed-off-by: Ling Samuel <lingsamuelgrace@gmail.com>
---
 cmd/minikube/cmd/node_start.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmd/minikube/cmd/node_start.go b/cmd/minikube/cmd/node_start.go
index a1711d1e1e52..e80964433e39 100644
--- a/cmd/minikube/cmd/node_start.go
+++ b/cmd/minikube/cmd/node_start.go
@@ -27,6 +27,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/mustload"
 	"k8s.io/minikube/pkg/minikube/node"
 	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/out/register"
 	"k8s.io/minikube/pkg/minikube/reason"
 	"k8s.io/minikube/pkg/minikube/style"
 )
@@ -54,6 +55,7 @@ var nodeStartCmd = &cobra.Command{
 			os.Exit(0)
 		}
 
+		register.Reg.SetStep(register.InitialSetup)
 		r, p, m, h, err := node.Provision(cc, n, false, viper.GetBool(deleteOnFailure))
 		if err != nil {
 			exit.Error(reason.GuestNodeProvision, "provisioning host for node", err)

From bbd0529f4bb71c7cd94995e87f92216de004284f Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Fri, 4 Dec 2020 12:12:31 +0000
Subject: [PATCH 34/45] fix GHReleases() to use release tag name instead of
 release name for release version

---
 hack/update/github.go | 2 +-
 hack/update/update.go | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/hack/update/github.go b/hack/update/github.go
index 8371296ddf23..897a36a40475 100644
--- a/hack/update/github.go
+++ b/hack/update/github.go
@@ -221,7 +221,7 @@ func GHReleases(ctx context.Context, owner, repo string) (stable, latest string,
 			return "", "", err
 		}
 		for _, rl := range rls {
-			ver := rl.GetName()
+			ver := rl.GetTagName()
 			if !semver.IsValid(ver) {
 				continue
 			}
diff --git a/hack/update/update.go b/hack/update/update.go
index 8cf74d01d829..dbf70619995b 100644
--- a/hack/update/update.go
+++ b/hack/update/update.go
@@ -88,8 +88,7 @@ func (i *Item) apply(data interface{}) error {
 	if i.Content == nil {
 		return fmt.Errorf("unable to update content: nothing to update")
 	}
-	org := string(i.Content)
-	str := org
+	str := string(i.Content)
 	for src, dst := range i.Replace {
 		out, err := ParseTmpl(dst, data, "")
 		if err != nil {

From 9f5d501509b7686cc4710da5d29af94887002134 Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Fri, 4 Dec 2020 12:31:10 +0000
Subject: [PATCH 35/45] add script to automate gopogh version update

---
 go.mod                                        |   2 +
 .../gopogh_version/update_gopogh_version.go   | 109 ++++++++++++++++++
 2 files changed, 111 insertions(+)
 create mode 100644 hack/update/gopogh_version/update_gopogh_version.go

diff --git a/go.mod b/go.mod
index a3d449a00a09..6df5ebf832ba 100644
--- a/go.mod
+++ b/go.mod
@@ -4,6 +4,7 @@ go 1.15
 
 require (
 	cloud.google.com/go/storage v1.10.0
+	contrib.go.opencensus.io/exporter/stackdriver v0.12.1
 	github.com/Azure/azure-sdk-for-go v42.3.0+incompatible
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.13.0
 	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
@@ -75,6 +76,7 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f
 	github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
+	go.opencensus.io v0.22.4
 	go.opentelemetry.io/otel v0.13.0
 	go.opentelemetry.io/otel/sdk v0.13.0
 	golang.org/x/build v0.0.0-20190927031335-2835ba2e683f
diff --git a/hack/update/gopogh_version/update_gopogh_version.go b/hack/update/gopogh_version/update_gopogh_version.go
new file mode 100644
index 000000000000..445bf0c49996
--- /dev/null
+++ b/hack/update/gopogh_version/update_gopogh_version.go
@@ -0,0 +1,109 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+Script expects the following env variables:
+ - UPDATE_TARGET=<string>: optional - if unset/absent, default option is "fs"; valid options are:
+   - "fs"  - update only local filesystem repo files [default]
+   - "gh"  - update only remote GitHub repo files and create PR (if one does not exist already)
+   - "all" - update local and remote repo files and create PR (if one does not exist already)
+ - GITHUB_TOKEN=<string>: GitHub [personal] access token
+   - note: GITHUB_TOKEN is required if UPDATE_TARGET is "gh" or "all"
+*/
+
+package main
+
+import (
+	"context"
+	"time"
+
+	"golang.org/x/mod/semver"
+	"k8s.io/klog/v2"
+
+	"k8s.io/minikube/hack/update"
+)
+
+const (
+	// default context timeout
+	cxTimeout = 300 * time.Second
+)
+
+var (
+	schema = map[string]update.Item{
+		".github/workflows/iso.yml": {
+			Replace: map[string]string{
+				`(?U)https://github.com/medyagh/gopogh/releases/download/.*/gopogh`: `https://github.com/medyagh/gopogh/releases/download/{{.StableVersion}}/gopogh`,
+			},
+		},
+		".github/workflows/kic_image.yml": {
+			Replace: map[string]string{
+				`(?U)https://github.com/medyagh/gopogh/releases/download/.*/gopogh`: `https://github.com/medyagh/gopogh/releases/download/{{.StableVersion}}/gopogh`,
+			},
+		},
+		".github/workflows/master.yml": {
+			Replace: map[string]string{
+				`(?U)https://github.com/medyagh/gopogh/releases/download/.*/gopogh`: `https://github.com/medyagh/gopogh/releases/download/{{.StableVersion}}/gopogh`,
+			},
+		},
+		".github/workflows/pr.yml": {
+			Replace: map[string]string{
+				`(?U)https://github.com/medyagh/gopogh/releases/download/.*/gopogh`: `https://github.com/medyagh/gopogh/releases/download/{{.StableVersion}}/gopogh`,
+			},
+		},
+		"hack/jenkins/common.sh": {
+			Replace: map[string]string{
+				`(?U)https://github.com/medyagh/gopogh/releases/download/.*/gopogh`: `https://github.com/medyagh/gopogh/releases/download/{{.StableVersion}}/gopogh`,
+			},
+		},
+	}
+
+	// PR data
+	prBranchPrefix = "update-gopogh-version_" // will be appended with first 7 characters of the PR commit SHA
+	prTitle        = `update_gopogh_version: {stable: "{{.StableVersion}}"}`
+	prIssue        = 9850
+)
+
+// Data holds stable gopogh version in semver format.
+type Data struct {
+	StableVersion string `json:"stableVersion"`
+}
+
+func main() {
+	// set a context with defined timeout
+	ctx, cancel := context.WithTimeout(context.Background(), cxTimeout)
+	defer cancel()
+
+	// get gopogh stable version from https://github.com/medyagh/gopogh
+	stable, err := gopoghVersion(ctx, "medyagh", "gopogh")
+	if err != nil || stable == "" {
+		klog.Fatalf("Unable to get gopogh stable version: %v", err)
+	}
+	stable = "v1.2.3"
+	data := Data{StableVersion: stable}
+	klog.Infof("gopogh stable version: %s", data.StableVersion)
+
+	update.Apply(ctx, schema, data, prBranchPrefix, prTitle, prIssue)
+}
+
+// gopoghVersion returns gopogh stable version in semver format.
+func gopoghVersion(ctx context.Context, owner, repo string) (stable string, err error) {
+	// get Kubernetes versions from GitHub Releases
+	stable, _, err = update.GHReleases(ctx, owner, repo)
+	if err != nil || !semver.IsValid(stable) {
+		return "", err
+	}
+	return stable, nil
+}

From c343113d6833d71df5e62b9b7b7eaa446f0f1be3 Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Fri, 4 Dec 2020 12:37:55 +0000
Subject: [PATCH 36/45] add script to automate gopogh version update

---
 hack/update/gopogh_version/update_gopogh_version.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hack/update/gopogh_version/update_gopogh_version.go b/hack/update/gopogh_version/update_gopogh_version.go
index 445bf0c49996..605fdff3b18a 100644
--- a/hack/update/gopogh_version/update_gopogh_version.go
+++ b/hack/update/gopogh_version/update_gopogh_version.go
@@ -91,7 +91,6 @@ func main() {
 	if err != nil || stable == "" {
 		klog.Fatalf("Unable to get gopogh stable version: %v", err)
 	}
-	stable = "v1.2.3"
 	data := Data{StableVersion: stable}
 	klog.Infof("gopogh stable version: %s", data.StableVersion)
 

From 934a0bf50f01c6a3bd552402b455b1914c75f12e Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Fri, 4 Dec 2020 12:40:18 +0000
Subject: [PATCH 37/45] add script to automate gopogh version update

---
 go.mod | 2 --
 1 file changed, 2 deletions(-)

diff --git a/go.mod b/go.mod
index 6df5ebf832ba..a3d449a00a09 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,6 @@ go 1.15
 
 require (
 	cloud.google.com/go/storage v1.10.0
-	contrib.go.opencensus.io/exporter/stackdriver v0.12.1
 	github.com/Azure/azure-sdk-for-go v42.3.0+incompatible
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.13.0
 	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
@@ -76,7 +75,6 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f
 	github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
-	go.opencensus.io v0.22.4
 	go.opentelemetry.io/otel v0.13.0
 	go.opentelemetry.io/otel/sdk v0.13.0
 	golang.org/x/build v0.0.0-20190927031335-2835ba2e683f

From e38a97c7182249b34e7b305b48427743c1db9775 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Fri, 4 Dec 2020 11:09:15 -0800
Subject: [PATCH 38/45] collect metrics for each container runtime

---
 hack/metrics/metrics.go | 68 ++++++++++++++++++++++-------------------
 1 file changed, 36 insertions(+), 32 deletions(-)

diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go
index 013a900d1633..b69520a32cd0 100644
--- a/hack/metrics/metrics.go
+++ b/hack/metrics/metrics.go
@@ -65,6 +65,10 @@ func execute() error {
 	if projectID == "" {
 		return fmt.Errorf("metrics collector requires a valid GCP project id set via the %s env variable", pkgtrace.ProjectEnvVar)
 	}
+	ctx := context.Background()
+	if err := downloadMinikube(ctx, tmpFile); err != nil {
+		return errors.Wrap(err, "downloading minikube")
+	}
 
 	// Register the view. It is imperative that this step exists,
 	// otherwise recorded metrics will be dropped and never exported.
@@ -77,44 +81,47 @@ func execute() error {
 	if err := view.Register(v); err != nil {
 		return errors.Wrap(err, "registering view")
 	}
+	for _, cr := range []string{"docker", "containerd", "crio"} {
+		sd, err := getExporter(projectID, cr)
+		if err != nil {
+			return errors.Wrap(err, "getting stackdriver exporter")
+		}
+		// Register it as a trace exporter
+		trace.RegisterExporter(sd)
+
+		if err := sd.StartMetricsExporter(); err != nil {
+			return errors.Wrap(err, "starting metric exporter")
+		}
+		// track minikube start time and record it to metrics collector
+		st, err := minikubeStartTime(ctx, projectID, tmpFile, cr)
+		if err != nil {
+			return errors.Wrap(err, "collecting start time")
+		}
+		fmt.Printf("Latency: %f\n", st)
+		stats.Record(ctx, latencyS.M(st))
+		time.Sleep(30 * time.Second)
+		sd.Flush()
+		sd.StopMetricsExporter()
+		trace.UnregisterExporter(sd)
+	}
+	return nil
+}
 
-	sd, err := stackdriver.NewExporter(stackdriver.Options{
+func getExporter(projectID, containerRuntime string) (*stackdriver.Exporter, error) {
+	return stackdriver.NewExporter(stackdriver.Options{
 		ProjectID: projectID,
 		// MetricPrefix helps uniquely identify your metrics.
 		MetricPrefix: "minikube_start",
 		// ReportingInterval sets the frequency of reporting metrics
 		// to stackdriver backend.
 		ReportingInterval:       1 * time.Second,
-		DefaultMonitoringLabels: getLabels(),
+		DefaultMonitoringLabels: getLabels(containerRuntime),
 	})
-	if err != nil {
-		return errors.Wrap(err, "getting stackdriver exporter")
-	}
-	// It is imperative to invoke flush before your main function exits
-	defer sd.Flush()
-
-	// Register it as a trace exporter
-	trace.RegisterExporter(sd)
-
-	if err := sd.StartMetricsExporter(); err != nil {
-		return errors.Wrap(err, "starting metric exporter")
-	}
-	defer sd.StopMetricsExporter()
-	ctx := context.Background()
-
-	// track minikube start time and record it to metrics collector
-	st, err := minikubeStartTime(ctx, projectID, tmpFile)
-	if err != nil {
-		return errors.Wrap(err, "collecting start time")
-	}
-	fmt.Printf("Latency: %f\n", st)
-	stats.Record(ctx, latencyS.M(st))
-	time.Sleep(30 * time.Second)
-	return nil
 }
 
-func getLabels() *stackdriver.Labels {
+func getLabels(containerRuntime string) *stackdriver.Labels {
 	l := &stackdriver.Labels{}
+	l.Set("container-runtime", containerRuntime, "container-runtime")
 	if labels == "" {
 		return l
 	}
@@ -130,13 +137,10 @@ func getLabels() *stackdriver.Labels {
 	return l
 }
 
-func minikubeStartTime(ctx context.Context, projectID, minikubePath string) (float64, error) {
-	if err := downloadMinikube(ctx, minikubePath); err != nil {
-		return 0, errors.Wrap(err, "downloading minikube")
-	}
+func minikubeStartTime(ctx context.Context, projectID, minikubePath, containerRuntime string) (float64, error) {
 	defer deleteMinikube(ctx, minikubePath)
 
-	cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile, "--memory=2000", "--trace=gcp")
+	cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile, "--memory=2000", "--trace=gcp", fmt.Sprintf("--container-runtime=%s", containerRuntime))
 	cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%s", pkgtrace.ProjectEnvVar, projectID))
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr

From ab1d2e474a8933d30f844dd36335427700092d5b Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Fri, 4 Dec 2020 11:36:16 -0800
Subject: [PATCH 39/45] don't fail if one container runtime fails

---
 hack/metrics/metrics.go | 45 ++++++++++++++++++++++++-----------------
 1 file changed, 26 insertions(+), 19 deletions(-)

diff --git a/hack/metrics/metrics.go b/hack/metrics/metrics.go
index b69520a32cd0..e3e85078e0e7 100644
--- a/hack/metrics/metrics.go
+++ b/hack/metrics/metrics.go
@@ -82,28 +82,35 @@ func execute() error {
 		return errors.Wrap(err, "registering view")
 	}
 	for _, cr := range []string{"docker", "containerd", "crio"} {
-		sd, err := getExporter(projectID, cr)
-		if err != nil {
-			return errors.Wrap(err, "getting stackdriver exporter")
+		if err := exportMinikubeStart(ctx, projectID, cr); err != nil {
+			log.Printf("error exporting minikube start data for runtime %v: %v", cr, err)
 		}
-		// Register it as a trace exporter
-		trace.RegisterExporter(sd)
+	}
+	return nil
+}
 
-		if err := sd.StartMetricsExporter(); err != nil {
-			return errors.Wrap(err, "starting metric exporter")
-		}
-		// track minikube start time and record it to metrics collector
-		st, err := minikubeStartTime(ctx, projectID, tmpFile, cr)
-		if err != nil {
-			return errors.Wrap(err, "collecting start time")
-		}
-		fmt.Printf("Latency: %f\n", st)
-		stats.Record(ctx, latencyS.M(st))
-		time.Sleep(30 * time.Second)
-		sd.Flush()
-		sd.StopMetricsExporter()
-		trace.UnregisterExporter(sd)
+func exportMinikubeStart(ctx context.Context, projectID, containerRuntime string) error {
+	sd, err := getExporter(projectID, containerRuntime)
+	if err != nil {
+		return errors.Wrap(err, "getting stackdriver exporter")
+	}
+	// Register it as a trace exporter
+	trace.RegisterExporter(sd)
+
+	if err := sd.StartMetricsExporter(); err != nil {
+		return errors.Wrap(err, "starting metric exporter")
+	}
+	// track minikube start time and record it to metrics collector
+	st, err := minikubeStartTime(ctx, projectID, tmpFile, containerRuntime)
+	if err != nil {
+		return errors.Wrap(err, "collecting start time")
 	}
+	fmt.Printf("Latency: %f\n", st)
+	stats.Record(ctx, latencyS.M(st))
+	time.Sleep(30 * time.Second)
+	sd.Flush()
+	sd.StopMetricsExporter()
+	trace.UnregisterExporter(sd)
 	return nil
 }
 

From 1b2076122e305f1523e1eb0b874160877a012863 Mon Sep 17 00:00:00 2001
From: Tharun <rajendrantharun@live.com>
Date: Fri, 27 Nov 2020 23:55:31 +0530
Subject: [PATCH 40/45] feat(minikube) display scheduledstop status in minikube
 status

Signed-off-by: Tharun <rajendrantharun@live.com>
---
 cmd/minikube/cmd/status.go      | 55 +++++++++++++++++++++++++++
 cmd/minikube/cmd/status_test.go | 67 +++++++++++++++++++++++++++++++++
 2 files changed, 122 insertions(+)

diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go
index 11950ee7d082..e35b2b46857e 100644
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@@ -170,6 +170,12 @@ type BaseState struct {
 	StepDetail string `json:",omitempty"`
 }
 
+// ScheduledStopStatus holds string representation of scheduledStopDuration
+type ScheduledStopStatus struct {
+	InitiatedTime string `json:",omitempty"`
+	ScheduledTime string `json:",omitempty"`
+}
+
 const (
 	minikubeNotRunningStatusFlag = 1 << 0
 	clusterNotRunningStatusFlag  = 1 << 1
@@ -187,6 +193,12 @@ type: Worker
 host: {{.Host}}
 kubelet: {{.Kubelet}}
 
+`
+
+	scheduledStopStatusFormat = `type: ScheduledDuration
+initiatedTime: {{.InitiatedTime}}
+scheduledTime: {{.ScheduledTime}}
+
 `
 )
 
@@ -256,6 +268,11 @@ func writeStatusesAtInterval(duration time.Duration, api libmachine.API, cc *con
 					exit.Error(reason.InternalStatusText, "status text failure", err)
 				}
 			}
+			if cc.ScheduledStop != nil {
+				if err := scheduledStopStatusText(cc.ScheduledStop, os.Stdout); err != nil {
+					exit.Error(reason.InternalStatusText, "status text failure", err)
+				}
+			}
 		case "json":
 			// Layout is currently only supported for JSON mode
 			if layout == "cluster" {
@@ -267,6 +284,11 @@ func writeStatusesAtInterval(duration time.Duration, api libmachine.API, cc *con
 					exit.Error(reason.InternalStatusJSON, "status json failure", err)
 				}
 			}
+			if cc.ScheduledStop != nil {
+				if err := scheduledStopStatusJSON(cc.ScheduledStop, os.Stdout); err != nil {
+					exit.Error(reason.InternalStatusJSON, "scheduledstop status json failure", err)
+				}
+			}
 		default:
 			exit.Message(reason.Usage, fmt.Sprintf("invalid output format: %s. Valid values: 'text', 'json'", output))
 		}
@@ -444,6 +466,39 @@ func statusJSON(st []*Status, w io.Writer) error {
 	return err
 }
 
+func scheduledStopStatusText(sd *config.ScheduledStopConfig, w io.Writer) error {
+	var scheduledStopStatus ScheduledStopStatus
+	if sd.InitiationTime == 0 {
+		scheduledStopStatus.InitiatedTime = "-"
+	} else {
+		scheduledStopStatus.InitiatedTime = time.Unix(sd.InitiationTime, 0).String()
+	}
+	if sd.Duration == 0 {
+		scheduledStopStatus.ScheduledTime = "-"
+	} else {
+		stopAt := time.Now().Add(sd.Duration).Unix()
+		scheduledStopStatus.ScheduledTime = time.Unix(stopAt, 0).String()
+	}
+	tmpl, err := template.New("scheduled-stop-status").Parse(scheduledStopStatusFormat)
+	if err != nil {
+		return err
+	}
+	if err := tmpl.Execute(w, scheduledStopStatus); err != nil {
+		return err
+	}
+	return nil
+}
+
+func scheduledStopStatusJSON(sd *config.ScheduledStopConfig, w io.Writer) error {
+	var js []byte
+	js, err := json.Marshal(sd)
+	if err != nil {
+		return err
+	}
+	_, err = w.Write(js)
+	return err
+}
+
 // readEventLog reads cloudevent logs from $MINIKUBE_HOME/profiles/<name>/events.json
 func readEventLog(name string) ([]cloudevents.Event, time.Time, error) {
 	path := localpath.EventLog(name)
diff --git a/cmd/minikube/cmd/status_test.go b/cmd/minikube/cmd/status_test.go
index 8bae78103706..dc3d9b6e6592 100644
--- a/cmd/minikube/cmd/status_test.go
+++ b/cmd/minikube/cmd/status_test.go
@@ -20,6 +20,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"testing"
+	"time"
+
+	"k8s.io/minikube/pkg/minikube/config"
 )
 
 func TestExitCode(t *testing.T) {
@@ -105,3 +108,67 @@ func TestStatusJSON(t *testing.T) {
 		})
 	}
 }
+
+func TestScheduledStopStatusText(t *testing.T) {
+	now := time.Now().Unix()
+	initiationTime := time.Unix(now, 0).String()
+
+	stopAt := time.Now().Add(time.Minute * 10).Unix()
+	scheduledTime := time.Unix(stopAt, 0).String()
+	var tests = []struct {
+		name  string
+		state *config.ScheduledStopConfig
+		want  string
+	}{
+		{
+			name:  "valid",
+			state: &config.ScheduledStopConfig{InitiationTime: now, Duration: time.Minute * 10},
+			want:  "type: ScheduledDuration\ninitiatedTime: " + initiationTime + "\nscheduledTime: " + scheduledTime + "\n\n",
+		},
+		{
+			name:  "missing",
+			state: &config.ScheduledStopConfig{},
+			want:  "type: ScheduledDuration\ninitiatedTime: -\nscheduledTime: -\n\n",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			var b bytes.Buffer
+			err := scheduledStopStatusText(tc.state, &b)
+			if err != nil {
+				t.Errorf("text(%+v) error: %v", tc.state, err)
+			}
+
+			got := b.String()
+			if got != tc.want {
+				t.Errorf("text(%+v) = %q, want: %q", tc.state, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestScheduledStopStatusJSON(t *testing.T) {
+	var tests = []struct {
+		name  string
+		state *config.ScheduledStopConfig
+	}{
+		{
+			name:  "valid",
+			state: &config.ScheduledStopConfig{InitiationTime: time.Now().Unix(), Duration: time.Minute * 5},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			var b bytes.Buffer
+			err := scheduledStopStatusJSON(tc.state, &b)
+			if err != nil {
+				t.Errorf("json(%+v) error: %v", tc.state, err)
+			}
+
+			st := &Status{}
+			if err := json.Unmarshal(b.Bytes(), st); err != nil {
+				t.Errorf("json(%+v) unmarshal error: %v", tc.state, err)
+			}
+		})
+	}
+}

From 81e27dd7b42c1d953f5628ce2a1f483fcbddd962 Mon Sep 17 00:00:00 2001
From: Tharun <rajendrantharun@live.com>
Date: Wed, 2 Dec 2020 16:50:02 +0530
Subject: [PATCH 41/45] fixed review feedbacks

Signed-off-by: Tharun <rajendrantharun@live.com>
---
 cmd/minikube/cmd/status.go              | 65 +++----------------
 cmd/minikube/cmd/status_test.go         | 85 +++----------------------
 site/content/en/docs/commands/status.md |  2 +-
 3 files changed, 19 insertions(+), 133 deletions(-)

diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go
index e35b2b46857e..c985c72157b3 100644
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@@ -135,6 +135,7 @@ type Status struct {
 	APIServer  string
 	Kubeconfig string
 	Worker     bool
+	TimeToStop string
 }
 
 // ClusterState holds a cluster state representation
@@ -170,12 +171,6 @@ type BaseState struct {
 	StepDetail string `json:",omitempty"`
 }
 
-// ScheduledStopStatus holds string representation of scheduledStopDuration
-type ScheduledStopStatus struct {
-	InitiatedTime string `json:",omitempty"`
-	ScheduledTime string `json:",omitempty"`
-}
-
 const (
 	minikubeNotRunningStatusFlag = 1 << 0
 	clusterNotRunningStatusFlag  = 1 << 1
@@ -186,6 +181,7 @@ host: {{.Host}}
 kubelet: {{.Kubelet}}
 apiserver: {{.APIServer}}
 kubeconfig: {{.Kubeconfig}}
+timeToStop: {{.TimeToStop}}
 
 `
 	workerStatusFormat = `{{.Name}}
@@ -193,12 +189,6 @@ type: Worker
 host: {{.Host}}
 kubelet: {{.Kubelet}}
 
-`
-
-	scheduledStopStatusFormat = `type: ScheduledDuration
-initiatedTime: {{.InitiatedTime}}
-scheduledTime: {{.ScheduledTime}}
-
 `
 )
 
@@ -268,11 +258,6 @@ func writeStatusesAtInterval(duration time.Duration, api libmachine.API, cc *con
 					exit.Error(reason.InternalStatusText, "status text failure", err)
 				}
 			}
-			if cc.ScheduledStop != nil {
-				if err := scheduledStopStatusText(cc.ScheduledStop, os.Stdout); err != nil {
-					exit.Error(reason.InternalStatusText, "status text failure", err)
-				}
-			}
 		case "json":
 			// Layout is currently only supported for JSON mode
 			if layout == "cluster" {
@@ -284,11 +269,6 @@ func writeStatusesAtInterval(duration time.Duration, api libmachine.API, cc *con
 					exit.Error(reason.InternalStatusJSON, "status json failure", err)
 				}
 			}
-			if cc.ScheduledStop != nil {
-				if err := scheduledStopStatusJSON(cc.ScheduledStop, os.Stdout); err != nil {
-					exit.Error(reason.InternalStatusJSON, "scheduledstop status json failure", err)
-				}
-			}
 		default:
 			exit.Message(reason.Usage, fmt.Sprintf("invalid output format: %s. Valid values: 'text', 'json'", output))
 		}
@@ -329,6 +309,7 @@ func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*St
 		Kubelet:    Nonexistent,
 		Kubeconfig: Nonexistent,
 		Worker:     !controlPlane,
+		TimeToStop: Nonexistent,
 	}
 
 	hs, err := machine.Status(api, name)
@@ -388,7 +369,12 @@ func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*St
 
 	stk := kverify.ServiceStatus(cr, "kubelet")
 	st.Kubelet = stk.String()
-
+	if cc.ScheduledStop != nil {
+		initiationTime := time.Unix(cc.ScheduledStop.InitiationTime, 0)
+		st.TimeToStop = time.Until(initiationTime.Add(cc.ScheduledStop.Duration)).String()
+	} else {
+		st.TimeToStop = Irrelevant
+	}
 	// Early exit for worker nodes
 	if !controlPlane {
 		return st, nil
@@ -466,39 +452,6 @@ func statusJSON(st []*Status, w io.Writer) error {
 	return err
 }
 
-func scheduledStopStatusText(sd *config.ScheduledStopConfig, w io.Writer) error {
-	var scheduledStopStatus ScheduledStopStatus
-	if sd.InitiationTime == 0 {
-		scheduledStopStatus.InitiatedTime = "-"
-	} else {
-		scheduledStopStatus.InitiatedTime = time.Unix(sd.InitiationTime, 0).String()
-	}
-	if sd.Duration == 0 {
-		scheduledStopStatus.ScheduledTime = "-"
-	} else {
-		stopAt := time.Now().Add(sd.Duration).Unix()
-		scheduledStopStatus.ScheduledTime = time.Unix(stopAt, 0).String()
-	}
-	tmpl, err := template.New("scheduled-stop-status").Parse(scheduledStopStatusFormat)
-	if err != nil {
-		return err
-	}
-	if err := tmpl.Execute(w, scheduledStopStatus); err != nil {
-		return err
-	}
-	return nil
-}
-
-func scheduledStopStatusJSON(sd *config.ScheduledStopConfig, w io.Writer) error {
-	var js []byte
-	js, err := json.Marshal(sd)
-	if err != nil {
-		return err
-	}
-	_, err = w.Write(js)
-	return err
-}
-
 // readEventLog reads cloudevent logs from $MINIKUBE_HOME/profiles/<name>/events.json
 func readEventLog(name string) ([]cloudevents.Event, time.Time, error) {
 	path := localpath.EventLog(name)
diff --git a/cmd/minikube/cmd/status_test.go b/cmd/minikube/cmd/status_test.go
index dc3d9b6e6592..aa9f905c22af 100644
--- a/cmd/minikube/cmd/status_test.go
+++ b/cmd/minikube/cmd/status_test.go
@@ -20,9 +20,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"testing"
-	"time"
-
-	"k8s.io/minikube/pkg/minikube/config"
 )
 
 func TestExitCode(t *testing.T) {
@@ -54,18 +51,18 @@ func TestStatusText(t *testing.T) {
 	}{
 		{
 			name:  "ok",
-			state: &Status{Name: "minikube", Host: "Running", Kubelet: "Running", APIServer: "Running", Kubeconfig: Configured},
-			want:  "minikube\ntype: Control Plane\nhost: Running\nkubelet: Running\napiserver: Running\nkubeconfig: Configured\n\n",
+			state: &Status{Name: "minikube", Host: "Running", Kubelet: "Running", APIServer: "Running", Kubeconfig: Configured, TimeToStop: "10m"},
+			want:  "minikube\ntype: Control Plane\nhost: Running\nkubelet: Running\napiserver: Running\nkubeconfig: Configured\ntimeToStop: 10m\n\n",
 		},
 		{
 			name:  "paused",
-			state: &Status{Name: "minikube", Host: "Running", Kubelet: "Stopped", APIServer: "Paused", Kubeconfig: Configured},
-			want:  "minikube\ntype: Control Plane\nhost: Running\nkubelet: Stopped\napiserver: Paused\nkubeconfig: Configured\n\n",
+			state: &Status{Name: "minikube", Host: "Running", Kubelet: "Stopped", APIServer: "Paused", Kubeconfig: Configured, TimeToStop: Nonexistent},
+			want:  "minikube\ntype: Control Plane\nhost: Running\nkubelet: Stopped\napiserver: Paused\nkubeconfig: Configured\ntimeToStop: Nonexistent\n\n",
 		},
 		{
 			name:  "down",
-			state: &Status{Name: "minikube", Host: "Stopped", Kubelet: "Stopped", APIServer: "Stopped", Kubeconfig: Misconfigured},
-			want:  "minikube\ntype: Control Plane\nhost: Stopped\nkubelet: Stopped\napiserver: Stopped\nkubeconfig: Misconfigured\n\n\nWARNING: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`\n",
+			state: &Status{Name: "minikube", Host: "Stopped", Kubelet: "Stopped", APIServer: "Stopped", Kubeconfig: Misconfigured, TimeToStop: Nonexistent},
+			want:  "minikube\ntype: Control Plane\nhost: Stopped\nkubelet: Stopped\napiserver: Stopped\nkubeconfig: Misconfigured\ntimeToStop: Nonexistent\n\n\nWARNING: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`\n",
 		},
 	}
 	for _, tc := range tests {
@@ -89,9 +86,9 @@ func TestStatusJSON(t *testing.T) {
 		name  string
 		state *Status
 	}{
-		{"ok", &Status{Host: "Running", Kubelet: "Running", APIServer: "Running", Kubeconfig: Configured}},
-		{"paused", &Status{Host: "Running", Kubelet: "Stopped", APIServer: "Paused", Kubeconfig: Configured}},
-		{"down", &Status{Host: "Stopped", Kubelet: "Stopped", APIServer: "Stopped", Kubeconfig: Misconfigured}},
+		{"ok", &Status{Host: "Running", Kubelet: "Running", APIServer: "Running", Kubeconfig: Configured, TimeToStop: "10m"}},
+		{"paused", &Status{Host: "Running", Kubelet: "Stopped", APIServer: "Paused", Kubeconfig: Configured, TimeToStop: Nonexistent}},
+		{"down", &Status{Host: "Stopped", Kubelet: "Stopped", APIServer: "Stopped", Kubeconfig: Misconfigured, TimeToStop: Nonexistent}},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
@@ -108,67 +105,3 @@ func TestStatusJSON(t *testing.T) {
 		})
 	}
 }
-
-func TestScheduledStopStatusText(t *testing.T) {
-	now := time.Now().Unix()
-	initiationTime := time.Unix(now, 0).String()
-
-	stopAt := time.Now().Add(time.Minute * 10).Unix()
-	scheduledTime := time.Unix(stopAt, 0).String()
-	var tests = []struct {
-		name  string
-		state *config.ScheduledStopConfig
-		want  string
-	}{
-		{
-			name:  "valid",
-			state: &config.ScheduledStopConfig{InitiationTime: now, Duration: time.Minute * 10},
-			want:  "type: ScheduledDuration\ninitiatedTime: " + initiationTime + "\nscheduledTime: " + scheduledTime + "\n\n",
-		},
-		{
-			name:  "missing",
-			state: &config.ScheduledStopConfig{},
-			want:  "type: ScheduledDuration\ninitiatedTime: -\nscheduledTime: -\n\n",
-		},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			var b bytes.Buffer
-			err := scheduledStopStatusText(tc.state, &b)
-			if err != nil {
-				t.Errorf("text(%+v) error: %v", tc.state, err)
-			}
-
-			got := b.String()
-			if got != tc.want {
-				t.Errorf("text(%+v) = %q, want: %q", tc.state, got, tc.want)
-			}
-		})
-	}
-}
-
-func TestScheduledStopStatusJSON(t *testing.T) {
-	var tests = []struct {
-		name  string
-		state *config.ScheduledStopConfig
-	}{
-		{
-			name:  "valid",
-			state: &config.ScheduledStopConfig{InitiationTime: time.Now().Unix(), Duration: time.Minute * 5},
-		},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			var b bytes.Buffer
-			err := scheduledStopStatusJSON(tc.state, &b)
-			if err != nil {
-				t.Errorf("json(%+v) error: %v", tc.state, err)
-			}
-
-			st := &Status{}
-			if err := json.Unmarshal(b.Bytes(), st); err != nil {
-				t.Errorf("json(%+v) unmarshal error: %v", tc.state, err)
-			}
-		})
-	}
-}
diff --git a/site/content/en/docs/commands/status.md b/site/content/en/docs/commands/status.md
index f4902dab0baf..e5c3c89e7dd5 100644
--- a/site/content/en/docs/commands/status.md
+++ b/site/content/en/docs/commands/status.md
@@ -23,7 +23,7 @@ minikube status [flags]
 
 ```
   -f, --format string         Go template format string for the status output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/
-                              For the list accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#Status (default "{{.Name}}\ntype: Control Plane\nhost: {{.Host}}\nkubelet: {{.Kubelet}}\napiserver: {{.APIServer}}\nkubeconfig: {{.Kubeconfig}}\n\n")
+                              For the list accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#Status (default "{{.Name}}\ntype: Control Plane\nhost: {{.Host}}\nkubelet: {{.Kubelet}}\napiserver: {{.APIServer}}\nkubeconfig: {{.Kubeconfig}}\ntimeToStop: {{.TimeToStop}}\n\n")
   -l, --layout string         output layout (EXPERIMENTAL, JSON only): 'nodes' or 'cluster' (default "nodes")
   -n, --node string           The node to check status for. Defaults to control plane. Leave blank with default format for status on all nodes.
   -o, --output string         minikube status --output OUTPUT. json, text (default "text")

From 3f57033c60d37abe6380dfd22c083fcf742d530e Mon Sep 17 00:00:00 2001
From: Tharun <rajendrantharun@live.com>
Date: Fri, 4 Dec 2020 13:05:39 +0530
Subject: [PATCH 42/45] added timetostop status in cluster layout

---
 cmd/minikube/cmd/status.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go
index c985c72157b3..2ce2f2064a71 100644
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@@ -372,8 +372,6 @@ func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*St
 	if cc.ScheduledStop != nil {
 		initiationTime := time.Unix(cc.ScheduledStop.InitiationTime, 0)
 		st.TimeToStop = time.Until(initiationTime.Add(cc.ScheduledStop.Duration)).String()
-	} else {
-		st.TimeToStop = Irrelevant
 	}
 	// Early exit for worker nodes
 	if !controlPlane {
@@ -486,6 +484,11 @@ func clusterState(sts []*Status) ClusterState {
 		statusName = sts[0].Host
 	}
 	sc := statusCode(statusName)
+
+	timeToStopStatusCode := Configured
+	if sts[0].TimeToStop == Nonexistent {
+		timeToStopStatusCode = Nonexistent
+	}
 	cs := ClusterState{
 		BinaryVersion: version.GetVersion(),
 
@@ -498,6 +501,7 @@ func clusterState(sts []*Status) ClusterState {
 
 		Components: map[string]BaseState{
 			"kubeconfig": {Name: "kubeconfig", StatusCode: statusCode(sts[0].Kubeconfig)},
+			"timetostop": {Name: "timetostop", StatusCode: statusCode(timeToStopStatusCode)},
 		},
 	}
 

From 35e2ce2dcfa147a6a161e72bb9478a5e1f2c0f39 Mon Sep 17 00:00:00 2001
From: Tharun <rajendrantharun@live.com>
Date: Fri, 4 Dec 2020 16:05:24 +0530
Subject: [PATCH 43/45] added check for timetostop status

---
 test/integration/scheduled_stop_test.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/integration/scheduled_stop_test.go b/test/integration/scheduled_stop_test.go
index 508a38599969..3de450e6299a 100644
--- a/test/integration/scheduled_stop_test.go
+++ b/test/integration/scheduled_stop_test.go
@@ -165,6 +165,10 @@ func ensureMinikubeStatus(ctx context.Context, t *testing.T, profile, wantStatus
 		if got != wantStatus {
 			return fmt.Errorf("expected post-stop host status to be -%q- but got *%q*", state.Stopped, got)
 		}
+		got = Status(ctx, t, Target(), profile, "TimeToStop", profile)
+		if got != "Nonexistent" {
+			return fmt.Errorf("expected post-stop timetostop status to be -%q- but got *%q*", "Nonexistent", got)
+		}
 		return nil
 	}
 	if err := retry.Expo(checkStatus, time.Second, time.Minute); err != nil {

From 4ca4491124b95d83982236b80bfa6b8cf4b8c74d Mon Sep 17 00:00:00 2001
From: Tharun <rajendrantharun@live.com>
Date: Sat, 5 Dec 2020 14:23:34 +0530
Subject: [PATCH 44/45] fixed review feedbacks on clusterstate field and tests

Signed-off-by: Tharun <rajendrantharun@live.com>
---
 cmd/minikube/cmd/status.go              |  8 +++-----
 test/integration/scheduled_stop_test.go | 17 +++++++++++++++++
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go
index 2ce2f2064a71..bfcde0880bb4 100644
--- a/cmd/minikube/cmd/status.go
+++ b/cmd/minikube/cmd/status.go
@@ -143,6 +143,7 @@ type ClusterState struct {
 	BaseState
 
 	BinaryVersion string
+	TimeToStop    string
 	Components    map[string]BaseState
 	Nodes         []NodeState
 }
@@ -485,10 +486,6 @@ func clusterState(sts []*Status) ClusterState {
 	}
 	sc := statusCode(statusName)
 
-	timeToStopStatusCode := Configured
-	if sts[0].TimeToStop == Nonexistent {
-		timeToStopStatusCode = Nonexistent
-	}
 	cs := ClusterState{
 		BinaryVersion: version.GetVersion(),
 
@@ -499,9 +496,10 @@ func clusterState(sts []*Status) ClusterState {
 			StatusDetail: codeDetails[sc],
 		},
 
+		TimeToStop: sts[0].TimeToStop,
+
 		Components: map[string]BaseState{
 			"kubeconfig": {Name: "kubeconfig", StatusCode: statusCode(sts[0].Kubeconfig)},
-			"timetostop": {Name: "timetostop", StatusCode: statusCode(timeToStopStatusCode)},
 		},
 	}
 
diff --git a/test/integration/scheduled_stop_test.go b/test/integration/scheduled_stop_test.go
index 3de450e6299a..81ee9e9e30f5 100644
--- a/test/integration/scheduled_stop_test.go
+++ b/test/integration/scheduled_stop_test.go
@@ -52,6 +52,11 @@ func TestScheduledStopWindows(t *testing.T) {
 	// schedule a stop for 5m from now
 	stopMinikube(ctx, t, profile, []string{"--schedule", "5m"})
 
+	scheduledStopMinikube(ctx, t, profile, "5m")
+	// sleep for 1 second
+	time.Sleep(time.Second)
+	// make sure timeToStop is present in status
+	checkTimetoStop(ctx, t, profile, "4m")
 	// make sure the systemd service is running
 	rr, err := Run(t, exec.CommandContext(ctx, Target(), []string{"ssh", "-p", profile, "--", "sudo", "systemctl", "show", constants.ScheduledStopSystemdService, "--no-page"}...))
 	if err != nil {
@@ -84,6 +89,11 @@ func TestScheduledStopUnix(t *testing.T) {
 
 	// schedule a stop for 5 min from now and make sure PID is created
 	stopMinikube(ctx, t, profile, []string{"--schedule", "5m"})
+	scheduledStopMinikube(ctx, t, profile, "5m")
+	// sleep for 1 second
+	time.Sleep(time.Second)
+	// make sure timeToStop is present in status
+	checkTimetoStop(ctx, t, profile, "4m")
 	pid := checkPID(t, profile)
 	if !processRunning(t, pid) {
 		t.Fatalf("process %v is not running", pid)
@@ -175,3 +185,10 @@ func ensureMinikubeStatus(ctx context.Context, t *testing.T, profile, wantStatus
 		t.Fatalf("error %v", err)
 	}
 }
+
+func checkTimetoStop(ctx context.Context, t *testing.T, profile string, currentTimeToStop string) {
+	got := Status(ctx, t, Target(), profile, "TimeToStop", profile)
+	if !strings.Contains(got, currentTimeToStop) {
+		t.Fatalf("expected timetostop status to be -%q- but got *%q*", currentTimeToStop, got)
+	}
+}

From d6a49a5beb7ea73f8b6f2fddece376c0fdb2b1e7 Mon Sep 17 00:00:00 2001
From: Tharun <rajendrantharun@live.com>
Date: Sat, 5 Dec 2020 14:49:56 +0530
Subject: [PATCH 45/45] updated scheduled_stop tests

Signed-off-by: Tharun <rajendrantharun@live.com>
---
 test/integration/scheduled_stop_test.go | 30 +++++++++----------------
 1 file changed, 10 insertions(+), 20 deletions(-)

diff --git a/test/integration/scheduled_stop_test.go b/test/integration/scheduled_stop_test.go
index 81ee9e9e30f5..58ed1a5d681a 100644
--- a/test/integration/scheduled_stop_test.go
+++ b/test/integration/scheduled_stop_test.go
@@ -51,12 +51,10 @@ func TestScheduledStopWindows(t *testing.T) {
 
 	// schedule a stop for 5m from now
 	stopMinikube(ctx, t, profile, []string{"--schedule", "5m"})
-
-	scheduledStopMinikube(ctx, t, profile, "5m")
 	// sleep for 1 second
 	time.Sleep(time.Second)
 	// make sure timeToStop is present in status
-	checkTimetoStop(ctx, t, profile, "4m")
+	ensureMinikubeStatus(ctx, t, profile, "TimeToStop", "4m")
 	// make sure the systemd service is running
 	rr, err := Run(t, exec.CommandContext(ctx, Target(), []string{"ssh", "-p", profile, "--", "sudo", "systemctl", "show", constants.ScheduledStopSystemdService, "--no-page"}...))
 	if err != nil {
@@ -72,7 +70,9 @@ func TestScheduledStopWindows(t *testing.T) {
 	// sleep for 5 seconds
 	time.Sleep(5 * time.Second)
 	// make sure minikube status is "Stopped"
-	ensureMinikubeStatus(ctx, t, profile, state.Stopped.String())
+	ensureMinikubeStatus(ctx, t, profile, "Host", state.Stopped.String())
+	// make sure minikube timtostop is "Nonexistent"
+	ensureMinikubeStatus(ctx, t, profile, "TimeToStop", "Nonexistent")
 }
 
 func TestScheduledStopUnix(t *testing.T) {
@@ -89,11 +89,10 @@ func TestScheduledStopUnix(t *testing.T) {
 
 	// schedule a stop for 5 min from now and make sure PID is created
 	stopMinikube(ctx, t, profile, []string{"--schedule", "5m"})
-	scheduledStopMinikube(ctx, t, profile, "5m")
 	// sleep for 1 second
 	time.Sleep(time.Second)
 	// make sure timeToStop is present in status
-	checkTimetoStop(ctx, t, profile, "4m")
+	ensureMinikubeStatus(ctx, t, profile, "TimeToStop", "4m")
 	pid := checkPID(t, profile)
 	if !processRunning(t, pid) {
 		t.Fatalf("process %v is not running", pid)
@@ -110,14 +109,16 @@ func TestScheduledStopUnix(t *testing.T) {
 	// sleep 12 just to be safe
 	stopMinikube(ctx, t, profile, []string{"--cancel-scheduled"})
 	time.Sleep(12 * time.Second)
-	ensureMinikubeStatus(ctx, t, profile, state.Running.String())
+	ensureMinikubeStatus(ctx, t, profile, "Host", state.Running.String())
 
 	// schedule another stop, make sure minikube status is "Stopped"
 	stopMinikube(ctx, t, profile, []string{"--schedule", "5s"})
 	if processRunning(t, pid) {
 		t.Fatalf("process %v running but should have been killed on reschedule of stop", pid)
 	}
-	ensureMinikubeStatus(ctx, t, profile, state.Stopped.String())
+	ensureMinikubeStatus(ctx, t, profile, "Host", state.Stopped.String())
+	// make sure minikube timtostop is "Nonexistent"
+	ensureMinikubeStatus(ctx, t, profile, "TimeToStop", "Nonexistent")
 }
 
 func startMinikube(ctx context.Context, t *testing.T, profile string) {
@@ -166,7 +167,7 @@ func processRunning(t *testing.T, pid string) bool {
 	t.Log("signal error was: ", err)
 	return err == nil
 }
-func ensureMinikubeStatus(ctx context.Context, t *testing.T, profile, wantStatus string) {
+func ensureMinikubeStatus(ctx context.Context, t *testing.T, profile, key string, wantStatus string) {
 	// wait allotted time to make sure minikube status is "Stopped"
 	checkStatus := func() error {
 		ctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Second))
@@ -175,20 +176,9 @@ func ensureMinikubeStatus(ctx context.Context, t *testing.T, profile, wantStatus
 		if got != wantStatus {
 			return fmt.Errorf("expected post-stop host status to be -%q- but got *%q*", state.Stopped, got)
 		}
-		got = Status(ctx, t, Target(), profile, "TimeToStop", profile)
-		if got != "Nonexistent" {
-			return fmt.Errorf("expected post-stop timetostop status to be -%q- but got *%q*", "Nonexistent", got)
-		}
 		return nil
 	}
 	if err := retry.Expo(checkStatus, time.Second, time.Minute); err != nil {
 		t.Fatalf("error %v", err)
 	}
 }
-
-func checkTimetoStop(ctx context.Context, t *testing.T, profile string, currentTimeToStop string) {
-	got := Status(ctx, t, Target(), profile, "TimeToStop", profile)
-	if !strings.Contains(got, currentTimeToStop) {
-		t.Fatalf("expected timetostop status to be -%q- but got *%q*", currentTimeToStop, got)
-	}
-}