Merge repositories.json after extracting preloaded tarball so that reference store isn't lost

go.mod

@@ -6,6 +6,7 @@ require (
 	cloud.google.com/go v0.45.1
 	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
 	github.com/Parallels/docker-machine-parallels v1.3.0
+	github.com/RaveNoX/go-jsonmerge v1.0.0 // indirect
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/blang/semver v3.5.0+incompatible
 	github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 // indirect
@@ -20,7 +21,7 @@ require (
 	github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5 // version is 0.7.1 to pin to a555e4f7a8f5
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
-	github.com/evanphx/json-patch v4.5.0+incompatible // indirect
+	github.com/evanphx/json-patch v4.5.0+incompatible
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
@@ -46,12 +47,14 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/libvirt/libvirt-go v3.4.0+incompatible
 	github.com/machine-drivers/docker-machine-driver-vmware v0.1.1
+	github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a
 	github.com/mattn/go-isatty v0.0.9
 	github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
 	github.com/moby/hyperkit v0.0.0-20171020124204-a12cd7250bcd
 	github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5
 	github.com/onsi/ginkgo v1.10.3 // indirect
 	github.com/onsi/gomega v1.7.1 // indirect
+	github.com/opencontainers/go-digest v1.0.0-rc1
 	github.com/otiai10/copy v1.0.2
 	github.com/pborman/uuid v1.2.0
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2

go.sum

@@ -52,6 +52,9 @@ github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
+github.com/RaveNoX/go-jsonmerge v1.0.0 h1:2e0nqnadoGUP8rAvcA0hkQelZreVO5X3BHomT2XMrAk=
+github.com/RaveNoX/go-jsonmerge v1.0.0/go.mod h1:qYM/NA77LhO4h51JJM7Z+xBU3ovqrNIACZe+SkSNVFo=
 github.com/Rican7/retry v0.1.0/go.mod h1:FgOROf8P5bebcC1DS0PdOQiqGUridaZvikzUmkFW6gg=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
@@ -93,6 +96,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
 github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU=
 github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 h1:+ziP/wVJWuAORkjv7386TRidVKY57X0bXBZFMeFlW+U=
@@ -429,6 +433,7 @@ github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c h1:3UvYABOQRhJAApj9MdCN
 github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA=
 github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d h1:hJXjZMxj0SWlMoQkzeZDLi2cmeiWKa7y1B8Rg+qaoEc=
 github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
+github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 h1:UUHMLvzt/31azWTN/ifGWef4WUqvXk0iRqdhdy/2uzI=
 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/juju/mutex v0.0.0-20180619145857-d21b13acf4bf h1:2d3cilQly1OpAfZcn4QRuwDOdVoHsM4cDTkcKbmO760=
@@ -493,6 +498,8 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
+github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a h1:+J2gw7Bw77w/fbK7wnNJJDKmw1IbWft2Ul5BzrG1Qm8=
+github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@@ -698,6 +705,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
 github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
 github.com/storageos/go-api v0.0.0-20180912212459-343b3eff91fc/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

pkg/minikube/cruntime/containerd.go

@@ -28,6 +28,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
@@ -310,6 +311,6 @@ func (r *Containerd) SystemLogCmd(len int) string {
 }
 
 // Preload preloads the container runtime with k8s images
-func (r *Containerd) Preload(k8sVersion string) error {
+func (r *Containerd) Preload(cfg config.KubernetesConfig) error {
 	return fmt.Errorf("not yet implemented for %s", r.Name())
 }

pkg/minikube/cruntime/crio.go

@@ -24,6 +24,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
@@ -227,6 +228,6 @@ func (r *CRIO) SystemLogCmd(len int) string {
 }
 
 // Preload preloads the container runtime with k8s images
-func (r *CRIO) Preload(k8sVersion string) error {
+func (r *CRIO) Preload(cfg config.KubernetesConfig) error {
 	return fmt.Errorf("not yet implemented for %s", r.Name())
 }

pkg/minikube/cruntime/cruntime.go

@@ -25,6 +25,7 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
@@ -100,7 +101,7 @@ type Manager interface {
 	// SystemLogCmd returns the command to return the system logs
 	SystemLogCmd(int) string
 	// Preload preloads the container runtime with k8s images
-	Preload(string) error
+	Preload(config.KubernetesConfig) error
 }
 
 // Config is runtime configuration

pkg/minikube/cruntime/docker.go

@@ -26,6 +26,10 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/docker"
 	"k8s.io/minikube/pkg/minikube/download"
 	"k8s.io/minikube/pkg/minikube/out"
 )
@@ -283,7 +287,24 @@ func (r *Docker) SystemLogCmd(len int) string {
 // 1. Copy over the preloaded tarball into the VM
 // 2. Extract the preloaded tarball to the correct directory
 // 3. Remove the tarball within the VM
-func (r *Docker) Preload(k8sVersion string) error {
+func (r *Docker) Preload(cfg config.KubernetesConfig) error {
+	k8sVersion := cfg.KubernetesVersion
+
+	// If images already exist, return
+	images, err := images.Kubeadm(cfg.ImageRepository, k8sVersion)
+	if err != nil {
+		return errors.Wrap(err, "getting images")
+	}
+	if DockerImagesPreloaded(r.Runner, images) {
+		glog.Info("Images already preloaded, skipping extraction")
+		return nil
+	}
+
+	refStore := docker.NewStorage(r.Runner)
+	if err := refStore.Save(); err != nil {
+		glog.Infof("error saving reference store: %v", err)
+	}
+
 	tarballPath := download.TarballPath(k8sVersion)
 	targetDir := "/"
 	targetName := "preloaded.tar.lz4"
@@ -314,5 +335,37 @@ func (r *Docker) Preload(k8sVersion string) error {
 	if err := r.Runner.Remove(fa); err != nil {
 		glog.Infof("error removing tarball: %v", err)
 	}
+
+	// save new reference store again
+	if err := refStore.Save(); err != nil {
+		glog.Infof("error saving reference store: %v", err)
+	}
+	// update reference store
+	if err := refStore.Update(); err != nil {
+		glog.Infof("error updating reference store: %v", err)
+	}
 	return r.Restart()
 }
+
+// DockerImagesPreloaded returns true if all images have been preloaded
+func DockerImagesPreloaded(runner command.Runner, images []string) bool {
+	rr, err := runner.RunCmd(exec.Command("docker", "images", "--format", "{{.Repository}}:{{.Tag}}"))
+	if err != nil {
+		return false
+	}
+	preloadedImages := map[string]struct{}{}
+	for _, i := range strings.Split(rr.Stdout.String(), "\n") {
+		preloadedImages[i] = struct{}{}
+	}
+
+	glog.Infof("Got preloaded images: %s", rr.Output())
+
+	// Make sure images == imgs
+	for _, i := range images {
+		if _, ok := preloadedImages[i]; !ok {
+			glog.Infof("%s wasn't preloaded", i)
+			return false
+		}
+	}
+	return true
+}

pkg/minikube/docker/store.go

@@ -0,0 +1,104 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package docker
+
+import (
+	"encoding/json"
+	"os/exec"
+	"path"
+
+	"github.com/golang/glog"
+	"github.com/opencontainers/go-digest"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/command"
+)
+
+const (
+	referenceStorePath = "/var/lib/docker/image/overlay2/repositories.json"
+)
+
+// Storage keeps track of reference stores
+type Storage struct {
+	refStores []ReferenceStore
+	runner    command.Runner
+}
+
+// ReferenceStore stores references to images in repositories.json
+// used by the docker daemon to name images
+// taken from "github.com/docker/docker/reference/store.go"
+type ReferenceStore struct {
+	Repositories map[string]repository
+}
+
+type repository map[string]digest.Digest
+
+// NewStorage returns a new storage type
+func NewStorage(runner command.Runner) *Storage {
+	return &Storage{
+		runner: runner,
+	}
+}
+
+// Save saves the current reference store in memory
+func (s *Storage) Save() error {
+	// get the contents of repositories.json in minikube
+	// if this command fails, assume the file doesn't exist
+	rr, err := s.runner.RunCmd(exec.Command("sudo", "cat", referenceStorePath))
+	if err != nil {
+		glog.Infof("repositories.json doesn't exist: %v", err)
+		return nil
+	}
+	contents := rr.Stdout.Bytes()
+	var rs ReferenceStore
+	if err := json.Unmarshal(contents, &rs); err != nil {
+		return err
+	}
+	s.refStores = append(s.refStores, rs)
+	return nil
+}
+
+// Update merges all reference stores and updates repositories.json
+func (s *Storage) Update() error {
+	// in case we didn't overwrite respoitories.json, do nothing
+	if len(s.refStores) == 1 {
+		return nil
+	}
+	// merge reference stores
+	merged := s.mergeReferenceStores()
+
+	// write to file in minikube
+	contents, err := json.Marshal(merged)
+	if err != nil {
+		return err
+	}
+
+	asset := assets.NewMemoryAsset(contents, path.Dir(referenceStorePath), path.Base(referenceStorePath), "0644")
+	return s.runner.Copy(asset)
+}
+
+func (s *Storage) mergeReferenceStores() ReferenceStore {
+	merged := ReferenceStore{
+		Repositories: map[string]repository{},
+	}
+	// otherwise, merge together reference stores
+	for _, rs := range s.refStores {
+		for k, v := range rs.Repositories {
+			merged.Repositories[k] = v
+		}
+	}
+	return merged
+}

pkg/minikube/docker/store_test.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package docker
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestMergeReferenceStores(t *testing.T) {
+	initial := ReferenceStore{
+		Repositories: map[string]repository{
+			"image1": repository{
+				"r1": "d1",
+				"r2": "d2",
+			},
+			"image2": repository{
+				"r1": "d1",
+				"r2": "d2",
+			},
+		},
+	}
+
+	afterPreload := ReferenceStore{
+		Repositories: map[string]repository{
+			"image1": repository{
+				"r1": "updated",
+				"r2": "updated",
+			},
+			"image3": repository{
+				"r3": "d3",
+			},
+		},
+	}
+
+	expected := ReferenceStore{
+		Repositories: map[string]repository{
+			"image1": repository{
+				"r1": "updated",
+				"r2": "updated",
+			},
+			"image2": repository{
+				"r1": "d1",
+				"r2": "d2",
+			},
+			"image3": repository{
+				"r3": "d3",
+			},
+		},
+	}
+
+	s := &Storage{
+		refStores: []ReferenceStore{initial, afterPreload},
+	}
+
+	actual := s.mergeReferenceStores()
+	if diff := cmp.Diff(actual, expected); diff != "" {
+		t.Errorf("Actual: %v, Expected: %v, Diff: %s", actual, expected, diff)
+	}
+}