diff --git a/go.mod b/go.mod index 49218c8804e2..92e5cb380a34 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( cloud.google.com/go v0.45.1 github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect github.com/Parallels/docker-machine-parallels v1.3.0 + github.com/RaveNoX/go-jsonmerge v1.0.0 // indirect github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/blang/semver v3.5.0+incompatible github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 // indirect @@ -20,7 +21,7 @@ require ( github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5 // version is 0.7.1 to pin to a555e4f7a8f5 github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect - github.com/evanphx/json-patch v4.5.0+incompatible // indirect + github.com/evanphx/json-patch v4.5.0+incompatible github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.3.1 // indirect github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 @@ -46,12 +47,14 @@ require ( github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/libvirt/libvirt-go v3.4.0+incompatible github.com/machine-drivers/docker-machine-driver-vmware v0.1.1 + github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a github.com/mattn/go-isatty v0.0.9 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 github.com/moby/hyperkit v0.0.0-20171020124204-a12cd7250bcd github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 github.com/onsi/ginkgo v1.10.3 // indirect github.com/onsi/gomega v1.7.1 // indirect + github.com/opencontainers/go-digest v1.0.0-rc1 github.com/otiai10/copy v1.0.2 github.com/pborman/uuid v1.2.0 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 diff --git a/go.sum b/go.sum index d4465220c136..153ab94a6947 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,9 @@ github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/RaveNoX/go-jsonmerge v1.0.0 h1:2e0nqnadoGUP8rAvcA0hkQelZreVO5X3BHomT2XMrAk= +github.com/RaveNoX/go-jsonmerge v1.0.0/go.mod h1:qYM/NA77LhO4h51JJM7Z+xBU3ovqrNIACZe+SkSNVFo= github.com/Rican7/retry v0.1.0/go.mod h1:FgOROf8P5bebcC1DS0PdOQiqGUridaZvikzUmkFW6gg= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= @@ -93,6 +96,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU= github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 h1:+ziP/wVJWuAORkjv7386TRidVKY57X0bXBZFMeFlW+U= @@ -429,6 +433,7 @@ github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c h1:3UvYABOQRhJAApj9MdCN github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA= github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d h1:hJXjZMxj0SWlMoQkzeZDLi2cmeiWKa7y1B8Rg+qaoEc= github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 h1:UUHMLvzt/31azWTN/ifGWef4WUqvXk0iRqdhdy/2uzI= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/mutex v0.0.0-20180619145857-d21b13acf4bf h1:2d3cilQly1OpAfZcn4QRuwDOdVoHsM4cDTkcKbmO760= @@ -493,6 +498,8 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= +github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a h1:+J2gw7Bw77w/fbK7wnNJJDKmw1IbWft2Ul5BzrG1Qm8= +github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -698,6 +705,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/storageos/go-api v0.0.0-20180912212459-343b3eff91fc/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 37d651bc3ca9..e023d7e77280 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -28,6 +28,7 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/bootstrapper/images" + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/out" ) @@ -310,6 +311,6 @@ func (r *Containerd) SystemLogCmd(len int) string { } // Preload preloads the container runtime with k8s images -func (r *Containerd) Preload(k8sVersion string) error { +func (r *Containerd) Preload(cfg config.KubernetesConfig) error { return fmt.Errorf("not yet implemented for %s", r.Name()) } diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 8b4be028cfb6..4b7e989fb4b5 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -24,6 +24,7 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/bootstrapper/images" + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/out" ) @@ -227,6 +228,6 @@ func (r *CRIO) SystemLogCmd(len int) string { } // Preload preloads the container runtime with k8s images -func (r *CRIO) Preload(k8sVersion string) error { +func (r *CRIO) Preload(cfg config.KubernetesConfig) error { return fmt.Errorf("not yet implemented for %s", r.Name()) } diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 468b84442970..ff9e96c39390 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -25,6 +25,7 @@ import ( "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/out" ) @@ -100,7 +101,7 @@ type Manager interface { // SystemLogCmd returns the command to return the system logs SystemLogCmd(int) string // Preload preloads the container runtime with k8s images - Preload(string) error + Preload(config.KubernetesConfig) error } // Config is runtime configuration diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 41d1385a3ed9..301e3ed05802 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -26,6 +26,10 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/bootstrapper/images" + "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/docker" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/out" ) @@ -283,7 +287,24 @@ func (r *Docker) SystemLogCmd(len int) string { // 1. Copy over the preloaded tarball into the VM // 2. Extract the preloaded tarball to the correct directory // 3. Remove the tarball within the VM -func (r *Docker) Preload(k8sVersion string) error { +func (r *Docker) Preload(cfg config.KubernetesConfig) error { + k8sVersion := cfg.KubernetesVersion + + // If images already exist, return + images, err := images.Kubeadm(cfg.ImageRepository, k8sVersion) + if err != nil { + return errors.Wrap(err, "getting images") + } + if DockerImagesPreloaded(r.Runner, images) { + glog.Info("Images already preloaded, skipping extraction") + return nil + } + + refStore := docker.NewStorage(r.Runner) + if err := refStore.Save(); err != nil { + glog.Infof("error saving reference store: %v", err) + } + tarballPath := download.TarballPath(k8sVersion) targetDir := "/" targetName := "preloaded.tar.lz4" @@ -314,5 +335,37 @@ func (r *Docker) Preload(k8sVersion string) error { if err := r.Runner.Remove(fa); err != nil { glog.Infof("error removing tarball: %v", err) } + + // save new reference store again + if err := refStore.Save(); err != nil { + glog.Infof("error saving reference store: %v", err) + } + // update reference store + if err := refStore.Update(); err != nil { + glog.Infof("error updating reference store: %v", err) + } return r.Restart() } + +// DockerImagesPreloaded returns true if all images have been preloaded +func DockerImagesPreloaded(runner command.Runner, images []string) bool { + rr, err := runner.RunCmd(exec.Command("docker", "images", "--format", "{{.Repository}}:{{.Tag}}")) + if err != nil { + return false + } + preloadedImages := map[string]struct{}{} + for _, i := range strings.Split(rr.Stdout.String(), "\n") { + preloadedImages[i] = struct{}{} + } + + glog.Infof("Got preloaded images: %s", rr.Output()) + + // Make sure images == imgs + for _, i := range images { + if _, ok := preloadedImages[i]; !ok { + glog.Infof("%s wasn't preloaded", i) + return false + } + } + return true +} diff --git a/pkg/minikube/docker/store.go b/pkg/minikube/docker/store.go new file mode 100644 index 000000000000..93a244edd157 --- /dev/null +++ b/pkg/minikube/docker/store.go @@ -0,0 +1,104 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package docker + +import ( + "encoding/json" + "os/exec" + "path" + + "github.com/golang/glog" + "github.com/opencontainers/go-digest" + "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/command" +) + +const ( + referenceStorePath = "/var/lib/docker/image/overlay2/repositories.json" +) + +// Storage keeps track of reference stores +type Storage struct { + refStores []ReferenceStore + runner command.Runner +} + +// ReferenceStore stores references to images in repositories.json +// used by the docker daemon to name images +// taken from "github.com/docker/docker/reference/store.go" +type ReferenceStore struct { + Repositories map[string]repository +} + +type repository map[string]digest.Digest + +// NewStorage returns a new storage type +func NewStorage(runner command.Runner) *Storage { + return &Storage{ + runner: runner, + } +} + +// Save saves the current reference store in memory +func (s *Storage) Save() error { + // get the contents of repositories.json in minikube + // if this command fails, assume the file doesn't exist + rr, err := s.runner.RunCmd(exec.Command("sudo", "cat", referenceStorePath)) + if err != nil { + glog.Infof("repositories.json doesn't exist: %v", err) + return nil + } + contents := rr.Stdout.Bytes() + var rs ReferenceStore + if err := json.Unmarshal(contents, &rs); err != nil { + return err + } + s.refStores = append(s.refStores, rs) + return nil +} + +// Update merges all reference stores and updates repositories.json +func (s *Storage) Update() error { + // in case we didn't overwrite respoitories.json, do nothing + if len(s.refStores) == 1 { + return nil + } + // merge reference stores + merged := s.mergeReferenceStores() + + // write to file in minikube + contents, err := json.Marshal(merged) + if err != nil { + return err + } + + asset := assets.NewMemoryAsset(contents, path.Dir(referenceStorePath), path.Base(referenceStorePath), "0644") + return s.runner.Copy(asset) +} + +func (s *Storage) mergeReferenceStores() ReferenceStore { + merged := ReferenceStore{ + Repositories: map[string]repository{}, + } + // otherwise, merge together reference stores + for _, rs := range s.refStores { + for k, v := range rs.Repositories { + merged.Repositories[k] = v + } + } + return merged +} diff --git a/pkg/minikube/docker/store_test.go b/pkg/minikube/docker/store_test.go new file mode 100644 index 000000000000..d9e9ce2e3cd6 --- /dev/null +++ b/pkg/minikube/docker/store_test.go @@ -0,0 +1,75 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package docker + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestMergeReferenceStores(t *testing.T) { + initial := ReferenceStore{ + Repositories: map[string]repository{ + "image1": repository{ + "r1": "d1", + "r2": "d2", + }, + "image2": repository{ + "r1": "d1", + "r2": "d2", + }, + }, + } + + afterPreload := ReferenceStore{ + Repositories: map[string]repository{ + "image1": repository{ + "r1": "updated", + "r2": "updated", + }, + "image3": repository{ + "r3": "d3", + }, + }, + } + + expected := ReferenceStore{ + Repositories: map[string]repository{ + "image1": repository{ + "r1": "updated", + "r2": "updated", + }, + "image2": repository{ + "r1": "d1", + "r2": "d2", + }, + "image3": repository{ + "r3": "d3", + }, + }, + } + + s := &Storage{ + refStores: []ReferenceStore{initial, afterPreload}, + } + + actual := s.mergeReferenceStores() + if diff := cmp.Diff(actual, expected); diff != "" { + t.Errorf("Actual: %v, Expected: %v, Diff: %s", actual, expected, diff) + } +} diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index ffa0aa7c186d..dc3349af4476 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -19,10 +19,8 @@ package machine import ( "fmt" "os" - "os/exec" "path" "path/filepath" - "strings" "sync" "time" @@ -66,7 +64,7 @@ func CacheImagesForBootstrapper(imageRepository string, version string, clusterB // LoadImages loads previously cached images into the container runtime func LoadImages(cc *config.ClusterConfig, runner command.Runner, images []string, cacheDir string) error { // Skip loading images if images already exist - if imagesPreloaded(runner, images) { + if cruntime.DockerImagesPreloaded(runner, images) { glog.Infof("Images are preloaded, skipping loading") return nil } @@ -108,28 +106,6 @@ func LoadImages(cc *config.ClusterConfig, runner command.Runner, images []string return nil } -func imagesPreloaded(runner command.Runner, images []string) bool { - rr, err := runner.RunCmd(exec.Command("docker", "images", "--format", "{{.Repository}}:{{.Tag}}")) - if err != nil { - return false - } - preloadedImages := map[string]struct{}{} - for _, i := range strings.Split(rr.Stdout.String(), "\n") { - preloadedImages[i] = struct{}{} - } - - glog.Infof("Got preloaded images: %s", rr.Output()) - - // Make sure images == imgs - for _, i := range images { - if _, ok := preloadedImages[i]; !ok { - glog.Infof("%s wasn't preloaded", i) - return false - } - } - return true -} - // needsTransfer returns an error if an image needs to be retransfered func needsTransfer(imgClient *client.Client, imgName string, cr cruntime.Manager) error { imgDgst := "" // for instance sha256:7c92a2c6bbcb6b6beff92d0a940779769c2477b807c202954c537e2e0deb9bed diff --git a/pkg/minikube/node/config.go b/pkg/minikube/node/config.go index d9500b276750..6ff1ee3db94a 100644 --- a/pkg/minikube/node/config.go +++ b/pkg/minikube/node/config.go @@ -67,9 +67,9 @@ func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s config disableOthers = false } - // Preload is overly invasive for bare metal, and caching is not meaningful. + // Preload is overly invasive for bare metal, and caching is not meaningful. KIC handled elsewhere. if driver.IsVM(drvName) { - if err := cr.Preload(k8s.KubernetesVersion); err != nil { + if err := cr.Preload(k8s); err != nil { switch err.(type) { case *cruntime.ErrISOFeature: out.T(out.Tip, "Existing disk is missing new features ({{.error}}). To upgrade, run 'minikube delete'", out.V{"error": err}) diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index a7b11c8f42e5..c891309a6939 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -166,6 +166,51 @@ func TestStartStop(t *testing.T) { }) } +func TestStartStopWithPreload(t *testing.T) { + if NoneDriver() { + t.Skipf("skipping %s - incompatible with none driver", t.Name()) + } + + profile := UniqueProfileName("test-preload") + ctx, cancel := context.WithTimeout(context.Background(), Minutes(40)) + defer CleanupWithLogs(t, profile, cancel) + + startArgs := []string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=3", "--wait=true"} + startArgs = append(startArgs, StartArgs()...) + k8sVersion := "v1.17.0" + startArgs = append(startArgs, fmt.Sprintf("--kubernetes-version=%s", k8sVersion)) + + rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...)) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + // Now, pull the busybox image into the VMs docker daemon + image := "busybox" + rr, err = Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "--", "docker", "pull", image)) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + // Restart minikube with v1.17.3, which has a preloaded tarball + startArgs = []string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=3", "--wait=true"} + startArgs = append(startArgs, StartArgs()...) + k8sVersion = "v1.17.3" + startArgs = append(startArgs, fmt.Sprintf("--kubernetes-version=%s", k8sVersion)) + rr, err = Run(t, exec.CommandContext(ctx, Target(), startArgs...)) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + // Ensure that busybox still exists in the daemon + rr, err = Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "--", "docker", "images")) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + if !strings.Contains(rr.Output(), image) { + t.Fatalf("Expected to find %s in output of `docker images`, instead got %s", image, rr.Output()) + } +} + // testPodScheduling asserts that this configuration can schedule new pods func testPodScheduling(ctx context.Context, t *testing.T, profile string) { t.Helper()