Skip to content

Commit 980e3ab

Browse files
medyaghmedyagh
authored
Merge pull request #6985 from priyawadhwa/no-overwrite-tar
Merge repositories.json after extracting preloaded tarball so that reference store isn't lost
2 parents ab3919d + e45b230 commit 980e3ab

File tree

11 files changed

+299
-32
lines changed

11 files changed

+299
-32
lines changed

‎go.mod

+4-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ require (
66
cloud.google.com/go v0.45.1
77
github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
88
github.com/Parallels/docker-machine-parallels v1.3.0
9+
github.com/RaveNoX/go-jsonmerge v1.0.0 // indirect
910
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
1011
github.com/blang/semver v3.5.0+incompatible
1112
github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 // indirect
@@ -20,7 +21,7 @@ require (
2021
github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5 // version is 0.7.1 to pin to a555e4f7a8f5
2122
github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f
2223
github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
23-
github.com/evanphx/json-patch v4.5.0+incompatible // indirect
24+
github.com/evanphx/json-patch v4.5.0+incompatible
2425
github.com/go-ole/go-ole v1.2.4 // indirect
2526
github.com/gogo/protobuf v1.3.1 // indirect
2627
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
@@ -46,12 +47,14 @@ require (
4647
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
4748
github.com/libvirt/libvirt-go v3.4.0+incompatible
4849
github.com/machine-drivers/docker-machine-driver-vmware v0.1.1
50+
github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a
4951
github.com/mattn/go-isatty v0.0.9
5052
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
5153
github.com/moby/hyperkit v0.0.0-20171020124204-a12cd7250bcd
5254
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5
5355
github.com/onsi/ginkgo v1.10.3 // indirect
5456
github.com/onsi/gomega v1.7.1 // indirect
57+
github.com/opencontainers/go-digest v1.0.0-rc1
5558
github.com/otiai10/copy v1.0.2
5659
github.com/pborman/uuid v1.2.0
5760
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2

‎go.sum

+8
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
5252
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
5353
github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
5454
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
55+
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
56+
github.com/RaveNoX/go-jsonmerge v1.0.0 h1:2e0nqnadoGUP8rAvcA0hkQelZreVO5X3BHomT2XMrAk=
57+
github.com/RaveNoX/go-jsonmerge v1.0.0/go.mod h1:qYM/NA77LhO4h51JJM7Z+xBU3ovqrNIACZe+SkSNVFo=
5558
github.com/Rican7/retry v0.1.0/go.mod h1:FgOROf8P5bebcC1DS0PdOQiqGUridaZvikzUmkFW6gg=
5659
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
5760
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
@@ -93,6 +96,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
9396
github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU=
9497
github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs=
9598
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
99+
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
96100
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
97101
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
98102
github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 h1:+ziP/wVJWuAORkjv7386TRidVKY57X0bXBZFMeFlW+U=
@@ -429,6 +433,7 @@ github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c h1:3UvYABOQRhJAApj9MdCN
429433
github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA=
430434
github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d h1:hJXjZMxj0SWlMoQkzeZDLi2cmeiWKa7y1B8Rg+qaoEc=
431435
github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
436+
github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
432437
github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 h1:UUHMLvzt/31azWTN/ifGWef4WUqvXk0iRqdhdy/2uzI=
433438
github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
434439
github.com/juju/mutex v0.0.0-20180619145857-d21b13acf4bf h1:2d3cilQly1OpAfZcn4QRuwDOdVoHsM4cDTkcKbmO760=
@@ -493,6 +498,8 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
493498
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
494499
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
495500
github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
501+
github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a h1:+J2gw7Bw77w/fbK7wnNJJDKmw1IbWft2Ul5BzrG1Qm8=
502+
github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0=
496503
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
497504
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
498505
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@@ -698,6 +705,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
698705
github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
699706
github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
700707
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
708+
github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
701709
github.com/storageos/go-api v0.0.0-20180912212459-343b3eff91fc/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY=
702710
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
703711
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

‎pkg/minikube/cruntime/containerd.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/golang/glog"
3030
"github.com/pkg/errors"
3131
"k8s.io/minikube/pkg/minikube/bootstrapper/images"
32+
"k8s.io/minikube/pkg/minikube/config"
3233
"k8s.io/minikube/pkg/minikube/out"
3334
)
3435

@@ -311,6 +312,6 @@ func (r *Containerd) SystemLogCmd(len int) string {
311312
}
312313

313314
// Preload preloads the container runtime with k8s images
314-
func (r *Containerd) Preload(k8sVersion string) error {
315+
func (r *Containerd) Preload(cfg config.KubernetesConfig) error {
315316
return fmt.Errorf("not yet implemented for %s", r.Name())
316317
}

‎pkg/minikube/cruntime/crio.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/golang/glog"
2626
"github.com/pkg/errors"
2727
"k8s.io/minikube/pkg/minikube/bootstrapper/images"
28+
"k8s.io/minikube/pkg/minikube/config"
2829
"k8s.io/minikube/pkg/minikube/out"
2930
)
3031

@@ -228,6 +229,6 @@ func (r *CRIO) SystemLogCmd(len int) string {
228229
}
229230

230231
// Preload preloads the container runtime with k8s images
231-
func (r *CRIO) Preload(k8sVersion string) error {
232+
func (r *CRIO) Preload(cfg config.KubernetesConfig) error {
232233
return fmt.Errorf("not yet implemented for %s", r.Name())
233234
}

‎pkg/minikube/cruntime/cruntime.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"github.com/pkg/errors"
2727
"k8s.io/minikube/pkg/minikube/assets"
2828
"k8s.io/minikube/pkg/minikube/command"
29+
"k8s.io/minikube/pkg/minikube/config"
2930
"k8s.io/minikube/pkg/minikube/out"
3031
)
3132

@@ -101,7 +102,7 @@ type Manager interface {
101102
// SystemLogCmd returns the command to return the system logs
102103
SystemLogCmd(int) string
103104
// Preload preloads the container runtime with k8s images
104-
Preload(string) error
105+
Preload(config.KubernetesConfig) error
105106
}
106107

107108
// Config is runtime configuration

‎pkg/minikube/cruntime/docker.go

+54-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ import (
2626
"github.com/golang/glog"
2727
"github.com/pkg/errors"
2828
"k8s.io/minikube/pkg/minikube/assets"
29+
"k8s.io/minikube/pkg/minikube/bootstrapper/images"
30+
"k8s.io/minikube/pkg/minikube/command"
31+
"k8s.io/minikube/pkg/minikube/config"
32+
"k8s.io/minikube/pkg/minikube/docker"
2933
"k8s.io/minikube/pkg/minikube/download"
3034
"k8s.io/minikube/pkg/minikube/out"
3135
)
@@ -283,7 +287,24 @@ func (r *Docker) SystemLogCmd(len int) string {
283287
// 1. Copy over the preloaded tarball into the VM
284288
// 2. Extract the preloaded tarball to the correct directory
285289
// 3. Remove the tarball within the VM
286-
func (r *Docker) Preload(k8sVersion string) error {
290+
func (r *Docker) Preload(cfg config.KubernetesConfig) error {
291+
k8sVersion := cfg.KubernetesVersion
292+
293+
// If images already exist, return
294+
images, err := images.Kubeadm(cfg.ImageRepository, k8sVersion)
295+
if err != nil {
296+
return errors.Wrap(err, "getting images")
297+
}
298+
if DockerImagesPreloaded(r.Runner, images) {
299+
glog.Info("Images already preloaded, skipping extraction")
300+
return nil
301+
}
302+
303+
refStore := docker.NewStorage(r.Runner)
304+
if err := refStore.Save(); err != nil {
305+
glog.Infof("error saving reference store: %v", err)
306+
}
307+
287308
tarballPath := download.TarballPath(k8sVersion)
288309
targetDir := "/"
289310
targetName := "preloaded.tar.lz4"
@@ -314,5 +335,37 @@ func (r *Docker) Preload(k8sVersion string) error {
314335
if err := r.Runner.Remove(fa); err != nil {
315336
glog.Infof("error removing tarball: %v", err)
316337
}
338+
339+
// save new reference store again
340+
if err := refStore.Save(); err != nil {
341+
glog.Infof("error saving reference store: %v", err)
342+
}
343+
// update reference store
344+
if err := refStore.Update(); err != nil {
345+
glog.Infof("error updating reference store: %v", err)
346+
}
317347
return r.Restart()
318348
}
349+
350+
// DockerImagesPreloaded returns true if all images have been preloaded
351+
func DockerImagesPreloaded(runner command.Runner, images []string) bool {
352+
rr, err := runner.RunCmd(exec.Command("docker", "images", "--format", "{{.Repository}}:{{.Tag}}"))
353+
if err != nil {
354+
return false
355+
}
356+
preloadedImages := map[string]struct{}{}
357+
for _, i := range strings.Split(rr.Stdout.String(), "\n") {
358+
preloadedImages[i] = struct{}{}
359+
}
360+
361+
glog.Infof("Got preloaded images: %s", rr.Output())
362+
363+
// Make sure images == imgs
364+
for _, i := range images {
365+
if _, ok := preloadedImages[i]; !ok {
366+
glog.Infof("%s wasn't preloaded", i)
367+
return false
368+
}
369+
}
370+
return true
371+
}

‎pkg/minikube/docker/store.go

+104
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/*
2+
Copyright 2020 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package docker
18+
19+
import (
20+
"encoding/json"
21+
"os/exec"
22+
"path"
23+
24+
"github.com/golang/glog"
25+
"github.com/opencontainers/go-digest"
26+
"k8s.io/minikube/pkg/minikube/assets"
27+
"k8s.io/minikube/pkg/minikube/command"
28+
)
29+
30+
const (
31+
referenceStorePath = "/var/lib/docker/image/overlay2/repositories.json"
32+
)
33+
34+
// Storage keeps track of reference stores
35+
type Storage struct {
36+
refStores []ReferenceStore
37+
runner command.Runner
38+
}
39+
40+
// ReferenceStore stores references to images in repositories.json
41+
// used by the docker daemon to name images
42+
// taken from "github.com/docker/docker/reference/store.go"
43+
type ReferenceStore struct {
44+
Repositories map[string]repository
45+
}
46+
47+
type repository map[string]digest.Digest
48+
49+
// NewStorage returns a new storage type
50+
func NewStorage(runner command.Runner) *Storage {
51+
return &Storage{
52+
runner: runner,
53+
}
54+
}
55+
56+
// Save saves the current reference store in memory
57+
func (s *Storage) Save() error {
58+
// get the contents of repositories.json in minikube
59+
// if this command fails, assume the file doesn't exist
60+
rr, err := s.runner.RunCmd(exec.Command("sudo", "cat", referenceStorePath))
61+
if err != nil {
62+
glog.Infof("repositories.json doesn't exist: %v", err)
63+
return nil
64+
}
65+
contents := rr.Stdout.Bytes()
66+
var rs ReferenceStore
67+
if err := json.Unmarshal(contents, &rs); err != nil {
68+
return err
69+
}
70+
s.refStores = append(s.refStores, rs)
71+
return nil
72+
}
73+
74+
// Update merges all reference stores and updates repositories.json
75+
func (s *Storage) Update() error {
76+
// in case we didn't overwrite respoitories.json, do nothing
77+
if len(s.refStores) == 1 {
78+
return nil
79+
}
80+
// merge reference stores
81+
merged := s.mergeReferenceStores()
82+
83+
// write to file in minikube
84+
contents, err := json.Marshal(merged)
85+
if err != nil {
86+
return err
87+
}
88+
89+
asset := assets.NewMemoryAsset(contents, path.Dir(referenceStorePath), path.Base(referenceStorePath), "0644")
90+
return s.runner.Copy(asset)
91+
}
92+
93+
func (s *Storage) mergeReferenceStores() ReferenceStore {
94+
merged := ReferenceStore{
95+
Repositories: map[string]repository{},
96+
}
97+
// otherwise, merge together reference stores
98+
for _, rs := range s.refStores {
99+
for k, v := range rs.Repositories {
100+
merged.Repositories[k] = v
101+
}
102+
}
103+
return merged
104+
}

‎pkg/minikube/docker/store_test.go

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
Copyright 2020 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package docker
18+
19+
import (
20+
"testing"
21+
22+
"github.com/google/go-cmp/cmp"
23+
)
24+
25+
func TestMergeReferenceStores(t *testing.T) {
26+
initial := ReferenceStore{
27+
Repositories: map[string]repository{
28+
"image1": repository{
29+
"r1": "d1",
30+
"r2": "d2",
31+
},
32+
"image2": repository{
33+
"r1": "d1",
34+
"r2": "d2",
35+
},
36+
},
37+
}
38+
39+
afterPreload := ReferenceStore{
40+
Repositories: map[string]repository{
41+
"image1": repository{
42+
"r1": "updated",
43+
"r2": "updated",
44+
},
45+
"image3": repository{
46+
"r3": "d3",
47+
},
48+
},
49+
}
50+
51+
expected := ReferenceStore{
52+
Repositories: map[string]repository{
53+
"image1": repository{
54+
"r1": "updated",
55+
"r2": "updated",
56+
},
57+
"image2": repository{
58+
"r1": "d1",
59+
"r2": "d2",
60+
},
61+
"image3": repository{
62+
"r3": "d3",
63+
},
64+
},
65+
}
66+
67+
s := &Storage{
68+
refStores: []ReferenceStore{initial, afterPreload},
69+
}
70+
71+
actual := s.mergeReferenceStores()
72+
if diff := cmp.Diff(actual, expected); diff != "" {
73+
t.Errorf("Actual: %v, Expected: %v, Diff: %s", actual, expected, diff)
74+
}
75+
}

0 commit comments

Comments
 (0)