From 0a0f39a112bcd06e091e1a5800d02be9de875b7a Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 2 Feb 2021 15:47:07 -0800 Subject: [PATCH 01/13] Implement minikube image load command --- cmd/minikube/cmd/image.go | 55 ++++++++++++++++++++++ cmd/minikube/cmd/root.go | 1 + pkg/minikube/image/image.go | 21 +++++++++ pkg/minikube/machine/cache_images.go | 70 ++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+) create mode 100644 cmd/minikube/cmd/image.go diff --git a/cmd/minikube/cmd/image.go b/cmd/minikube/cmd/image.go new file mode 100644 index 000000000000..d8748d29255b --- /dev/null +++ b/cmd/minikube/cmd/image.go @@ -0,0 +1,55 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "github.com/spf13/cobra" + "github.com/spf13/viper" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/machine" + "k8s.io/minikube/pkg/minikube/reason" +) + +// imageCmd represents the image command +var imageCmd = &cobra.Command{ + Use: "image", + Short: "Load a local image into minikube", + Long: "Load a local image into minikube", +} + +// loadImageCmd represents the image load command +var loadImageCmd = &cobra.Command{ + Use: "load", + Short: "Load a local image into minikube", + Long: "Load a local image into minikube", + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + exit.Message(reason.Usage, "Please provide an image in your local daemon or a path to an image tarball to load into minikube via ") + } + // Cache and load images into docker daemon + profile := viper.GetString(config.ProfileName) + img := args[0] + if err := machine.LoadImage(profile, img); err != nil { + exit.Error(reason.InternalCacheLoad, "Failed to cache and load images", err) + } + }, +} + +func init() { + imageCmd.AddCommand(loadImageCmd) +} diff --git a/cmd/minikube/cmd/root.go b/cmd/minikube/cmd/root.go index a29886b5b207..17850e985b82 100644 --- a/cmd/minikube/cmd/root.go +++ b/cmd/minikube/cmd/root.go @@ -201,6 +201,7 @@ func init() { dockerEnvCmd, podmanEnvCmd, cacheCmd, + imageCmd, }, }, { diff --git a/pkg/minikube/image/image.go b/pkg/minikube/image/image.go index ece5fb4e324d..f736e0b4b37d 100644 --- a/pkg/minikube/image/image.go +++ b/pkg/minikube/image/image.go @@ -123,6 +123,27 @@ func LoadFromTarball(binary, img string) error { } +// SaveToTarball saves img as a tarball at the given path +func SaveToTarball(img, path string) error { + if !ExistsImageInDaemon(img) { + return fmt.Errorf("%s does not exist in local daemon, can't save to tarball", img) + } + ref, err := name.ParseReference(img) + if err != nil { + return errors.Wrap(err, "parsing reference") + } + i, err := daemon.Image(ref) + if err != nil { + return errors.Wrap(err, "getting image") + } + f, err := os.Create(path) + if err != nil { + return errors.Wrap(err, "creating tmp path") + } + defer f.Close() + return tarball.Write(ref, i, f) +} + // Tag returns just the image with the tag // eg image:tag@sha256:digest -> image:tag if there is an associated tag // if not possible, just return the initial img diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index 3b5f2d379e53..90731e7859f6 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -18,7 +18,9 @@ package machine import ( "fmt" + "io/ioutil" "os" + "os/exec" "path" "path/filepath" "strings" @@ -61,6 +63,74 @@ func CacheImagesForBootstrapper(imageRepository string, version string, clusterB return nil } +// LoadImage loads the local image into the container runtime +func LoadImage(profile, img string) error { + cc, err := config.Load(profile) + if err != nil { + return errors.Wrap(err, "loading profile") + } + // if the image exists in the local daemon, save it as a tarball + if image.ExistsImageInDaemon(img) { + tmpFile, err := ioutil.TempFile("", "") + if err != nil { + return errors.Wrap(err, "temp file") + } + tmpFile.Close() + defer os.Remove(tmpFile.Name()) + img = tmpFile.Name() + } + dst := "/tmp/img.tar" + if err := copyAndLoadTarballIntoHost(cc, img, dst, profile); err != nil { + return errors.Wrap(err, "copying tarball into host") + } + return nil +} + +func copyAndLoadTarballIntoHost(cc *config.ClusterConfig, srcPath, dstPath, profile string) error { + c, err := config.Load(profile) + if err != nil { + return errors.Wrap(err, "loading profile config") + } + api, err := NewAPIClient() + if err != nil { + return errors.Wrap(err, "api") + } + defer api.Close() + for _, n := range c.Nodes { + m := config.MachineName(*c, n) + h, err := api.Load(m) + if err != nil { + return errors.Wrap(err, "loading api") + } + cr, err := CommandRunner(h) + if err != nil { + return errors.Wrap(err, "command runner") + } + tarballImg, err := assets.NewFileAsset(srcPath, filepath.Dir(dstPath), filepath.Base(dstPath), "0644") + if err != nil { + return errors.Wrap(err, "new file asset") + } + // copy tarball into minikube + + if err := cr.Copy(tarballImg); err != nil { + return errors.Wrap(err, "copying tarball") + } + // load image into container runtime + containerRuntime, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: cr}) + if err != nil { + return errors.Wrap(err, "runtime") + } + if err := containerRuntime.LoadImage(dstPath); err != nil { + return errors.Wrap(err, "loading image into container runtime") + } + // delete destination image tarball on host + if _, err := cr.RunCmd(exec.Command("rm", dstPath)); err != nil { + return errors.Wrap(err, "removing destination tarball") + } + } + return nil +} + // LoadImages loads previously cached images into the container runtime func LoadImages(cc *config.ClusterConfig, runner command.Runner, images []string, cacheDir string) error { cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: runner}) From 1d63f085e4836c509e34313fcfd648ab4778ef34 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 4 Feb 2021 14:45:27 -0800 Subject: [PATCH 02/13] Add command --- cmd/minikube/cmd/image.go | 2 +- pkg/minikube/machine/cache_images.go | 74 +++++++++++----------------- pkg/minikube/mustload/mustload.go | 2 +- pkg/minikube/reason/reason.go | 1 + 4 files changed, 33 insertions(+), 46 deletions(-) diff --git a/cmd/minikube/cmd/image.go b/cmd/minikube/cmd/image.go index d8748d29255b..76f0e0dd217c 100644 --- a/cmd/minikube/cmd/image.go +++ b/cmd/minikube/cmd/image.go @@ -45,7 +45,7 @@ var loadImageCmd = &cobra.Command{ profile := viper.GetString(config.ProfileName) img := args[0] if err := machine.LoadImage(profile, img); err != nil { - exit.Error(reason.InternalCacheLoad, "Failed to cache and load images", err) + exit.Error(reason.GuestImageLoad, "Failed to load image", err) } }, } diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index 90731e7859f6..e15222a28569 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -20,7 +20,6 @@ import ( "fmt" "io/ioutil" "os" - "os/exec" "path" "path/filepath" "strings" @@ -65,69 +64,56 @@ func CacheImagesForBootstrapper(imageRepository string, version string, clusterB // LoadImage loads the local image into the container runtime func LoadImage(profile, img string) error { - cc, err := config.Load(profile) + // save image in temporary dir + tmpDir, err := ioutil.TempDir("", "") if err != nil { - return errors.Wrap(err, "loading profile") - } - // if the image exists in the local daemon, save it as a tarball - if image.ExistsImageInDaemon(img) { - tmpFile, err := ioutil.TempFile("", "") - if err != nil { - return errors.Wrap(err, "temp file") - } - tmpFile.Close() - defer os.Remove(tmpFile.Name()) - img = tmpFile.Name() + return errors.Wrap(err, "temp dir") } - dst := "/tmp/img.tar" - if err := copyAndLoadTarballIntoHost(cc, img, dst, profile); err != nil { - return errors.Wrap(err, "copying tarball into host") - } - return nil -} + defer os.Remove(tmpDir) + imgPath := filepath.Join(tmpDir, img) + imgPath = localpath.SanitizeCacheDir(imgPath) -func copyAndLoadTarballIntoHost(cc *config.ClusterConfig, srcPath, dstPath, profile string) error { - c, err := config.Load(profile) - if err != nil { - return errors.Wrap(err, "loading profile config") + // save the docker image to tarball + if err := image.SaveToTarball(img, imgPath); err != nil { + return errors.Wrapf(err, "saving %s to tarball", img) } + api, err := NewAPIClient() if err != nil { return errors.Wrap(err, "api") } defer api.Close() + + c, err := config.Load(profile) + if err != nil { + return errors.Wrap(err, "loading profile") + } + for _, n := range c.Nodes { m := config.MachineName(*c, n) - h, err := api.Load(m) + status, err := Status(api, m) if err != nil { - return errors.Wrap(err, "loading api") + klog.Warningf("error getting status for %s: %v", m, err) + continue } - cr, err := CommandRunner(h) - if err != nil { - return errors.Wrap(err, "command runner") + if status != state.Running.String() { // the not running hosts will load on next start + continue } - tarballImg, err := assets.NewFileAsset(srcPath, filepath.Dir(dstPath), filepath.Base(dstPath), "0644") + h, err := api.Load(m) if err != nil { - return errors.Wrap(err, "new file asset") - } - // copy tarball into minikube - - if err := cr.Copy(tarballImg); err != nil { - return errors.Wrap(err, "copying tarball") + klog.Warningf("Failed to load machine %q: %v", m, err) + continue } - // load image into container runtime - containerRuntime, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: cr}) + cr, err := CommandRunner(h) if err != nil { - return errors.Wrap(err, "runtime") - } - if err := containerRuntime.LoadImage(dstPath); err != nil { - return errors.Wrap(err, "loading image into container runtime") + klog.Warningf("Failed to get command runner %q: %v", m, err) + continue } - // delete destination image tarball on host - if _, err := cr.RunCmd(exec.Command("rm", dstPath)); err != nil { - return errors.Wrap(err, "removing destination tarball") + if err := LoadImages(c, cr, []string{img}, tmpDir); err != nil { + klog.Warningf("Failed to load %s into node %s, skipping and trying other nodes: %v", img, n.Name, err) } } + return nil } diff --git a/pkg/minikube/mustload/mustload.go b/pkg/minikube/mustload/mustload.go index 934ab33373b6..eae1b272068a 100644 --- a/pkg/minikube/mustload/mustload.go +++ b/pkg/minikube/mustload/mustload.go @@ -113,7 +113,7 @@ func Running(name string) ClusterController { host, err := machine.LoadHost(api, name) if err != nil { - exit.Error(reason.GuestLoadHost, "Unable to load host", err) + exit.Error(reason.GuestImageLoad, "Unable to load host", err) } cr, err := machine.CommandRunner(host) diff --git a/pkg/minikube/reason/reason.go b/pkg/minikube/reason/reason.go index ee406919d95b..7010168d27d3 100644 --- a/pkg/minikube/reason/reason.go +++ b/pkg/minikube/reason/reason.go @@ -238,6 +238,7 @@ var ( GuestCert = Kind{ID: "GUEST_CERT", ExitCode: ExGuestError} GuestCpConfig = Kind{ID: "GUEST_CP_CONFIG", ExitCode: ExGuestConfig} GuestDeletion = Kind{ID: "GUEST_DELETION", ExitCode: ExGuestError} + GuestImageLoad = Kind{ID: "GUEST_IMAGE_LOAD", ExitCode: ExGuestError} GuestLoadHost = Kind{ID: "GUEST_LOAD_HOST", ExitCode: ExGuestError} GuestMount = Kind{ID: "GUEST_MOUNT", ExitCode: ExGuestError} GuestMountConflict = Kind{ID: "GUEST_MOUNT_CONFLICT", ExitCode: ExGuestConflict} From 07562be3f0af2c6cc778ef644f1415f4ed5e45be Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 4 Feb 2021 14:47:28 -0800 Subject: [PATCH 03/13] clean up code so we aren't duplicating it --- pkg/minikube/image/image.go | 21 --------------------- pkg/minikube/machine/cache_images.go | 8 ++------ 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/pkg/minikube/image/image.go b/pkg/minikube/image/image.go index f736e0b4b37d..ece5fb4e324d 100644 --- a/pkg/minikube/image/image.go +++ b/pkg/minikube/image/image.go @@ -123,27 +123,6 @@ func LoadFromTarball(binary, img string) error { } -// SaveToTarball saves img as a tarball at the given path -func SaveToTarball(img, path string) error { - if !ExistsImageInDaemon(img) { - return fmt.Errorf("%s does not exist in local daemon, can't save to tarball", img) - } - ref, err := name.ParseReference(img) - if err != nil { - return errors.Wrap(err, "parsing reference") - } - i, err := daemon.Image(ref) - if err != nil { - return errors.Wrap(err, "getting image") - } - f, err := os.Create(path) - if err != nil { - return errors.Wrap(err, "creating tmp path") - } - defer f.Close() - return tarball.Write(ref, i, f) -} - // Tag returns just the image with the tag // eg image:tag@sha256:digest -> image:tag if there is an associated tag // if not possible, just return the initial img diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index e15222a28569..5a86336ad6eb 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -70,12 +70,8 @@ func LoadImage(profile, img string) error { return errors.Wrap(err, "temp dir") } defer os.Remove(tmpDir) - imgPath := filepath.Join(tmpDir, img) - imgPath = localpath.SanitizeCacheDir(imgPath) - - // save the docker image to tarball - if err := image.SaveToTarball(img, imgPath); err != nil { - return errors.Wrapf(err, "saving %s to tarball", img) + if err := image.SaveToDir([]string{img}, tmpDir); err != nil { + return errors.Wrap(err, "save to dir") } api, err := NewAPIClient() From f365fcc692d098e1a3e2137251e4dab2bc358fcb Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 4 Feb 2021 16:21:40 -0800 Subject: [PATCH 04/13] Add integration test to ensure works correctly --- test/integration/load_test.go | 71 +++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 test/integration/load_test.go diff --git a/test/integration/load_test.go b/test/integration/load_test.go new file mode 100644 index 000000000000..a3322d2ef202 --- /dev/null +++ b/test/integration/load_test.go @@ -0,0 +1,71 @@ +// +build integration + +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "fmt" + "os/exec" + "strings" + "testing" +) + +func TestImageLoad(t *testing.T) { + if NoneDriver() { + t.Skip("skipping on none driver") + } + profile := UniqueProfileName("load-image") + ctx, _ := context.WithTimeout(context.Background(), Minutes(5)) + // defer CleanupWithLogs(t, profile, cancel) + + args := append([]string{"start", "-p", profile, "--memory=2000"}, StartArgs()...) + rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) + if err != nil { + t.Fatalf("starting minikube: %v\n%s", err, rr.Output()) + } + + // pull busybox + busybox := "busybox:latest" + rr, err = Run(t, exec.CommandContext(ctx, "docker", "pull", busybox)) + if err != nil { + t.Fatalf("starting minikube: %v\n%s", err, rr.Output()) + } + + // tag busybox + newImage := fmt.Sprintf("busybox:%s", profile) + rr, err = Run(t, exec.CommandContext(ctx, "docker", "tag", busybox, newImage)) + if err != nil { + t.Fatalf("starting minikube: %v\n%s", err, rr.Output()) + } + + // try to load the new image into minikube + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", newImage)) + if err != nil { + t.Fatalf("loading image into minikube: %v\n%s", err, rr.Output()) + } + + // make sure the image was correctly loaded + rr, err = Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "--", "sudo", "ctr", "-n=k8s.io", "image", "ls")) + if err != nil { + t.Fatalf("listing images: %v\n%s", err, rr.Output()) + } + if !strings.Contains(rr.Output(), newImage) { + t.Fatalf("expected %s to be loaded into minikube but the image is not there", newImage) + } +} From d257ff2a94233830657abf20a4c31414add8a879 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 4 Feb 2021 16:22:27 -0800 Subject: [PATCH 05/13] remove debugging comments --- test/integration/load_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/load_test.go b/test/integration/load_test.go index a3322d2ef202..8d49cceaedb9 100644 --- a/test/integration/load_test.go +++ b/test/integration/load_test.go @@ -31,8 +31,8 @@ func TestImageLoad(t *testing.T) { t.Skip("skipping on none driver") } profile := UniqueProfileName("load-image") - ctx, _ := context.WithTimeout(context.Background(), Minutes(5)) - // defer CleanupWithLogs(t, profile, cancel) + ctx, cancel := context.WithTimeout(context.Background(), Minutes(5)) + defer CleanupWithLogs(t, profile, cancel) args := append([]string{"start", "-p", profile, "--memory=2000"}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) From 97d78f846efb317c14d2bd57ae09dd7331fe32b5 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 4 Feb 2021 16:23:39 -0800 Subject: [PATCH 06/13] Fix mustload --- pkg/minikube/mustload/mustload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/mustload/mustload.go b/pkg/minikube/mustload/mustload.go index eae1b272068a..934ab33373b6 100644 --- a/pkg/minikube/mustload/mustload.go +++ b/pkg/minikube/mustload/mustload.go @@ -113,7 +113,7 @@ func Running(name string) ClusterController { host, err := machine.LoadHost(api, name) if err != nil { - exit.Error(reason.GuestImageLoad, "Unable to load host", err) + exit.Error(reason.GuestLoadHost, "Unable to load host", err) } cr, err := machine.CommandRunner(host) From 63d3a20cf8811d1958d8e235b37a755ff52a23c8 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 4 Feb 2021 16:31:44 -0800 Subject: [PATCH 07/13] update docs --- site/content/en/docs/commands/image.md | 106 +++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 site/content/en/docs/commands/image.md diff --git a/site/content/en/docs/commands/image.md b/site/content/en/docs/commands/image.md new file mode 100644 index 000000000000..e98ceb45ebb7 --- /dev/null +++ b/site/content/en/docs/commands/image.md @@ -0,0 +1,106 @@ +--- +title: "image" +description: > + Load a local image into minikube +--- + + +## minikube image + +Load a local image into minikube + +### Synopsis + +Load a local image into minikube + +### Options inherited from parent commands + +``` + --add_dir_header If true, adds the file directory to the header of the log messages + --alsologtostderr log to standard error as well as files + -b, --bootstrapper string The name of the cluster bootstrapper that will set up the Kubernetes cluster. (default "kubeadm") + -h, --help + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_file string If non-empty, use this log file + --log_file_max_size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) + --logtostderr log to standard error instead of files + --one_output If true, only write logs to their native severity level (vs also writing to each lower severity level + -p, --profile string The name of the minikube VM being used. This can be set to allow having multiple instances of minikube independently. (default "minikube") + --skip_headers If true, avoid header prefixes in the log messages + --skip_log_headers If true, avoid headers when opening log files + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + --user string Specifies the user executing the operation. Useful for auditing operations executed by 3rd party tools. Defaults to the operating system username. + -v, --v Level number for the log level verbosity + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + +## minikube image help + +Help about any command + +### Synopsis + +Help provides help for any command in the application. +Simply type image help [path to command] for full details. + +```shell +minikube image help [command] [flags] +``` + +### Options inherited from parent commands + +``` + --add_dir_header If true, adds the file directory to the header of the log messages + --alsologtostderr log to standard error as well as files + -b, --bootstrapper string The name of the cluster bootstrapper that will set up the Kubernetes cluster. (default "kubeadm") + -h, --help + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_file string If non-empty, use this log file + --log_file_max_size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) + --logtostderr log to standard error instead of files + --one_output If true, only write logs to their native severity level (vs also writing to each lower severity level + -p, --profile string The name of the minikube VM being used. This can be set to allow having multiple instances of minikube independently. (default "minikube") + --skip_headers If true, avoid header prefixes in the log messages + --skip_log_headers If true, avoid headers when opening log files + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + --user string Specifies the user executing the operation. Useful for auditing operations executed by 3rd party tools. Defaults to the operating system username. + -v, --v Level number for the log level verbosity + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + +## minikube image load + +Load a local image into minikube + +### Synopsis + +Load a local image into minikube + +```shell +minikube image load [flags] +``` + +### Options inherited from parent commands + +``` + --add_dir_header If true, adds the file directory to the header of the log messages + --alsologtostderr log to standard error as well as files + -b, --bootstrapper string The name of the cluster bootstrapper that will set up the Kubernetes cluster. (default "kubeadm") + -h, --help + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_file string If non-empty, use this log file + --log_file_max_size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) + --logtostderr log to standard error instead of files + --one_output If true, only write logs to their native severity level (vs also writing to each lower severity level + -p, --profile string The name of the minikube VM being used. This can be set to allow having multiple instances of minikube independently. (default "minikube") + --skip_headers If true, avoid header prefixes in the log messages + --skip_log_headers If true, avoid headers when opening log files + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + --user string Specifies the user executing the operation. Useful for auditing operations executed by 3rd party tools. Defaults to the operating system username. + -v, --v Level number for the log level verbosity + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + From 22a65a0c12bc83b9d1c44072ad9c8382e9a30d20 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 5 Feb 2021 10:47:44 -0800 Subject: [PATCH 08/13] Add --all flag to --- cmd/minikube/cmd/cache.go | 33 +++++++++++++++- cmd/minikube/cmd/image.go | 9 +++-- pkg/minikube/machine/cache_images.go | 58 +--------------------------- pkg/minikube/node/cache.go | 4 +- pkg/minikube/node/start.go | 8 +++- 5 files changed, 46 insertions(+), 66 deletions(-) diff --git a/cmd/minikube/cmd/cache.go b/cmd/minikube/cmd/cache.go index ff1ab11568d9..3e4e855bee02 100644 --- a/cmd/minikube/cmd/cache.go +++ b/cmd/minikube/cmd/cache.go @@ -18,17 +18,25 @@ package cmd import ( "github.com/spf13/cobra" + "github.com/spf13/viper" + "k8s.io/klog/v2" cmdConfig "k8s.io/minikube/cmd/minikube/cmd/config" + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/image" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/node" + "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/reason" ) // cacheImageConfigKey is the config field name used to store which images we have previously cached const cacheImageConfigKey = "cache" +var ( + all string +) + // cacheCmd represents the cache command var cacheCmd = &cobra.Command{ Use: "cache", @@ -43,7 +51,7 @@ var addCacheCmd = &cobra.Command{ Long: "Add an image to local cache.", Run: func(cmd *cobra.Command, args []string) { // Cache and load images into docker daemon - if err := machine.CacheAndLoadImages(args); err != nil { + if err := machine.CacheAndLoadImages(args, cacheAddProfiles()); err != nil { exit.Error(reason.InternalCacheLoad, "Failed to cache and load images", err) } // Add images to config file @@ -53,6 +61,27 @@ var addCacheCmd = &cobra.Command{ }, } +func addCacheCmdFlags() { + addCacheCmd.Flags().Bool(all, false, "Add image to cache for all running minikube clusters") +} + +func cacheAddProfiles() []*config.Profile { + if viper.GetBool(all) { + validProfiles, _, err := config.ListProfiles() // need to load image to all profiles + if err != nil { + klog.Warningf("error listing profiles: %v", err) + } + return validProfiles + } else { + profile := viper.GetString(config.ProfileName) + p, err := config.LoadProfile(profile) + if err != nil { + exit.Message(reason.Usage, "{{.profile}} profile is not valid: {{.err}}", out.V{"profile": profile, "err": err}) + } + return []*config.Profile{p} + } +} + // deleteCacheCmd represents the cache delete command var deleteCacheCmd = &cobra.Command{ Use: "delete", @@ -76,7 +105,7 @@ var reloadCacheCmd = &cobra.Command{ Short: "reload cached images.", Long: "reloads images previously added using the 'cache add' subcommand", Run: func(cmd *cobra.Command, args []string) { - err := node.CacheAndLoadImagesInConfig() + err := node.CacheAndLoadImagesInConfig(cacheAddProfiles()) if err != nil { exit.Error(reason.GuestCacheLoad, "Failed to reload cached images", err) } diff --git a/cmd/minikube/cmd/image.go b/cmd/minikube/cmd/image.go index 76f0e0dd217c..3186c7120a76 100644 --- a/cmd/minikube/cmd/image.go +++ b/cmd/minikube/cmd/image.go @@ -39,12 +39,15 @@ var loadImageCmd = &cobra.Command{ Long: "Load a local image into minikube", Run: func(cmd *cobra.Command, args []string) { if len(args) == 0 { - exit.Message(reason.Usage, "Please provide an image in your local daemon or a path to an image tarball to load into minikube via ") + exit.Message(reason.Usage, "Please provide an image in your local daemon to load into minikube via ") } // Cache and load images into docker daemon - profile := viper.GetString(config.ProfileName) + profile, err := config.LoadProfile(viper.GetString(config.ProfileName)) + if err != nil { + exit.Error(reason.Usage, "loading profile", err) + } img := args[0] - if err := machine.LoadImage(profile, img); err != nil { + if err := machine.CacheAndLoadImages([]string{img}, []*config.Profile{profile}); err != nil { exit.Error(reason.GuestImageLoad, "Failed to load image", err) } }, diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index 5a86336ad6eb..1288e7117bf2 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -18,7 +18,6 @@ package machine import ( "fmt" - "io/ioutil" "os" "path" "path/filepath" @@ -62,57 +61,6 @@ func CacheImagesForBootstrapper(imageRepository string, version string, clusterB return nil } -// LoadImage loads the local image into the container runtime -func LoadImage(profile, img string) error { - // save image in temporary dir - tmpDir, err := ioutil.TempDir("", "") - if err != nil { - return errors.Wrap(err, "temp dir") - } - defer os.Remove(tmpDir) - if err := image.SaveToDir([]string{img}, tmpDir); err != nil { - return errors.Wrap(err, "save to dir") - } - - api, err := NewAPIClient() - if err != nil { - return errors.Wrap(err, "api") - } - defer api.Close() - - c, err := config.Load(profile) - if err != nil { - return errors.Wrap(err, "loading profile") - } - - for _, n := range c.Nodes { - m := config.MachineName(*c, n) - status, err := Status(api, m) - if err != nil { - klog.Warningf("error getting status for %s: %v", m, err) - continue - } - if status != state.Running.String() { // the not running hosts will load on next start - continue - } - h, err := api.Load(m) - if err != nil { - klog.Warningf("Failed to load machine %q: %v", m, err) - continue - } - cr, err := CommandRunner(h) - if err != nil { - klog.Warningf("Failed to get command runner %q: %v", m, err) - continue - } - if err := LoadImages(c, cr, []string{img}, tmpDir); err != nil { - klog.Warningf("Failed to load %s into node %s, skipping and trying other nodes: %v", img, n.Name, err) - } - } - - return nil -} - // LoadImages loads previously cached images into the container runtime func LoadImages(cc *config.ClusterConfig, runner command.Runner, images []string, cacheDir string) error { cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: runner}) @@ -210,7 +158,7 @@ func needsTransfer(imgClient *client.Client, imgName string, cr cruntime.Manager } // CacheAndLoadImages caches and loads images to all profiles -func CacheAndLoadImages(images []string) error { +func CacheAndLoadImages(images []string, profiles []*config.Profile) error { if len(images) == 0 { return nil } @@ -225,10 +173,6 @@ func CacheAndLoadImages(images []string) error { return errors.Wrap(err, "api") } defer api.Close() - profiles, _, err := config.ListProfiles() // need to load image to all profiles - if err != nil { - return errors.Wrap(err, "list profiles") - } succeeded := []string{} failed := []string{} diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index e775ed3ca104..0abcb26aa09a 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -210,7 +210,7 @@ func saveImagesToTarFromConfig() error { // CacheAndLoadImagesInConfig loads the images currently in the config file // called by 'start' and 'cache reload' commands. -func CacheAndLoadImagesInConfig() error { +func CacheAndLoadImagesInConfig(profiles []*config.Profile) error { images, err := imagesInConfigFile() if err != nil { return errors.Wrap(err, "images") @@ -218,7 +218,7 @@ func CacheAndLoadImagesInConfig() error { if len(images) == 0 { return nil } - return machine.CacheAndLoadImages(images) + return machine.CacheAndLoadImages(images, profiles) } func imagesInConfigFile() ([]string, error) { diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index d1081fd7bb70..3dbbb6fff956 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -145,10 +145,14 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { wg.Add(1) go func() { - if err := CacheAndLoadImagesInConfig(); err != nil { + defer wg.Done() + profile, err := config.LoadProfile(starter.Cfg.Name) + if err != nil { + out.FailureT("Unable to load profile: {{.error}}", out.V{"error": err}) + } + if err := CacheAndLoadImagesInConfig([]*config.Profile{profile}); err != nil { out.FailureT("Unable to push cached images: {{.error}}", out.V{"error": err}) } - wg.Done() }() // enable addons, both old and new! From 9a9c160b3f1d4ea3133b37eecce6107260a159c8 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 5 Feb 2021 14:26:44 -0800 Subject: [PATCH 09/13] Move load test into functional test --- cmd/minikube/cmd/cache.go | 14 +++--- test/integration/functional_test.go | 35 ++++++++++++++ test/integration/load_test.go | 71 ----------------------------- 3 files changed, 42 insertions(+), 78 deletions(-) delete mode 100644 test/integration/load_test.go diff --git a/cmd/minikube/cmd/cache.go b/cmd/minikube/cmd/cache.go index 3e4e855bee02..50d57ca9aaf8 100644 --- a/cmd/minikube/cmd/cache.go +++ b/cmd/minikube/cmd/cache.go @@ -72,14 +72,13 @@ func cacheAddProfiles() []*config.Profile { klog.Warningf("error listing profiles: %v", err) } return validProfiles - } else { - profile := viper.GetString(config.ProfileName) - p, err := config.LoadProfile(profile) - if err != nil { - exit.Message(reason.Usage, "{{.profile}} profile is not valid: {{.err}}", out.V{"profile": profile, "err": err}) - } - return []*config.Profile{p} } + profile := viper.GetString(config.ProfileName) + p, err := config.LoadProfile(profile) + if err != nil { + exit.Message(reason.Usage, "{{.profile}} profile is not valid: {{.err}}", out.V{"profile": profile, "err": err}) + } + return []*config.Profile{p} } // deleteCacheCmd represents the cache delete command @@ -113,6 +112,7 @@ var reloadCacheCmd = &cobra.Command{ } func init() { + addCacheCmdFlags() cacheCmd.AddCommand(addCacheCmd) cacheCmd.AddCommand(deleteCacheCmd) cacheCmd.AddCommand(reloadCacheCmd) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index d52ae2f864c7..300ac20b65fe 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -127,6 +127,7 @@ func TestFunctional(t *testing.T) { {"UpdateContextCmd", validateUpdateContextCmd}, {"DockerEnv", validateDockerEnv}, {"NodeLabels", validateNodeLabels}, + {"LoadImage", validateLoadImage}, } for _, tc := range tests { tc := tc @@ -158,6 +159,40 @@ func validateNodeLabels(ctx context.Context, t *testing.T, profile string) { } } +// validateLoadImage makes sure that `minikube load image` works as expected +func validateLoadImage(ctx context.Context, t *testing.T, profile string) { + defer PostMortemLogs(t, profile) + + // pull busybox + busybox := "busybox:latest" + rr, err := Run(t, exec.CommandContext(ctx, "docker", "pull", busybox)) + if err != nil { + t.Fatalf("starting minikube: %v\n%s", err, rr.Output()) + } + + // tag busybox + newImage := fmt.Sprintf("busybox:%s", profile) + rr, err = Run(t, exec.CommandContext(ctx, "docker", "tag", busybox, newImage)) + if err != nil { + t.Fatalf("starting minikube: %v\n%s", err, rr.Output()) + } + + // try to load the new image into minikube + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", newImage)) + if err != nil { + t.Fatalf("loading image into minikube: %v\n%s", err, rr.Output()) + } + + // make sure the image was correctly loaded + rr, err = Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "--", "sudo", "ctr", "-n=k8s.io", "image", "ls")) + if err != nil { + t.Fatalf("listing images: %v\n%s", err, rr.Output()) + } + if !strings.Contains(rr.Output(), newImage) { + t.Fatalf("expected %s to be loaded into minikube but the image is not there", newImage) + } +} + // check functionality of minikube after evaling docker-env // TODO: Add validatePodmanEnv for crio runtime: #10231 func validateDockerEnv(ctx context.Context, t *testing.T, profile string) { diff --git a/test/integration/load_test.go b/test/integration/load_test.go deleted file mode 100644 index 8d49cceaedb9..000000000000 --- a/test/integration/load_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// +build integration - -/* -Copyright 2020 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package integration - -import ( - "context" - "fmt" - "os/exec" - "strings" - "testing" -) - -func TestImageLoad(t *testing.T) { - if NoneDriver() { - t.Skip("skipping on none driver") - } - profile := UniqueProfileName("load-image") - ctx, cancel := context.WithTimeout(context.Background(), Minutes(5)) - defer CleanupWithLogs(t, profile, cancel) - - args := append([]string{"start", "-p", profile, "--memory=2000"}, StartArgs()...) - rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) - if err != nil { - t.Fatalf("starting minikube: %v\n%s", err, rr.Output()) - } - - // pull busybox - busybox := "busybox:latest" - rr, err = Run(t, exec.CommandContext(ctx, "docker", "pull", busybox)) - if err != nil { - t.Fatalf("starting minikube: %v\n%s", err, rr.Output()) - } - - // tag busybox - newImage := fmt.Sprintf("busybox:%s", profile) - rr, err = Run(t, exec.CommandContext(ctx, "docker", "tag", busybox, newImage)) - if err != nil { - t.Fatalf("starting minikube: %v\n%s", err, rr.Output()) - } - - // try to load the new image into minikube - rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", newImage)) - if err != nil { - t.Fatalf("loading image into minikube: %v\n%s", err, rr.Output()) - } - - // make sure the image was correctly loaded - rr, err = Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "--", "sudo", "ctr", "-n=k8s.io", "image", "ls")) - if err != nil { - t.Fatalf("listing images: %v\n%s", err, rr.Output()) - } - if !strings.Contains(rr.Output(), newImage) { - t.Fatalf("expected %s to be loaded into minikube but the image is not there", newImage) - } -} From 15548fc83e811164d065a5366d3cc7508e0420b4 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 5 Feb 2021 14:36:55 -0800 Subject: [PATCH 10/13] update docs --- site/content/en/docs/commands/cache.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/site/content/en/docs/commands/cache.md b/site/content/en/docs/commands/cache.md index 6f7eff26a93c..39065402f700 100644 --- a/site/content/en/docs/commands/cache.md +++ b/site/content/en/docs/commands/cache.md @@ -47,6 +47,12 @@ Add an image to local cache. minikube cache add [flags] ``` +### Options + +``` + -- Add image to cache for all running minikube clusters +``` + ### Options inherited from parent commands ``` From 97c58bf83928140f89d9289c17209a23475b2413 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 5 Feb 2021 15:29:43 -0800 Subject: [PATCH 11/13] Fix test --- test/integration/functional_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 300ac20b65fe..60a18bcdd9a4 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -161,7 +161,7 @@ func validateNodeLabels(ctx context.Context, t *testing.T, profile string) { // validateLoadImage makes sure that `minikube load image` works as expected func validateLoadImage(ctx context.Context, t *testing.T, profile string) { - defer PostMortemLogs(t, profile) + // defer PostMortemLogs(t, profile) // pull busybox busybox := "busybox:latest" @@ -184,7 +184,13 @@ func validateLoadImage(ctx context.Context, t *testing.T, profile string) { } // make sure the image was correctly loaded - rr, err = Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "--", "sudo", "ctr", "-n=k8s.io", "image", "ls")) + var cmd *exec.Cmd + if ContainerRuntime() == "docker" { + cmd = exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "--", "docker", "images", "--format", "{{.Repository}}:{{.Tag}}") + } else { + cmd = exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "--", "sudo", "ctr", "-n=k8s.io", "image", "ls") + } + rr, err = Run(t, cmd) if err != nil { t.Fatalf("listing images: %v\n%s", err, rr.Output()) } From 53887f407ca50b1b7d6c4ab89af9a4c3a6239d30 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 8 Feb 2021 10:30:17 -0800 Subject: [PATCH 12/13] skip load image on none driver --- test/integration/functional_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 60a18bcdd9a4..aca25d07215a 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -161,8 +161,10 @@ func validateNodeLabels(ctx context.Context, t *testing.T, profile string) { // validateLoadImage makes sure that `minikube load image` works as expected func validateLoadImage(ctx context.Context, t *testing.T, profile string) { - // defer PostMortemLogs(t, profile) - + if NoneDriver() { + t.Skip("load image not available on none driver") + } + defer PostMortemLogs(t, profile) // pull busybox busybox := "busybox:latest" rr, err := Run(t, exec.CommandContext(ctx, "docker", "pull", busybox)) From e8945f293448b3a4c84875ee12e4c81b314c2daa Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 9 Feb 2021 13:36:40 -0800 Subject: [PATCH 13/13] Add warning for cache add --- cmd/minikube/cmd/cache.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/minikube/cmd/cache.go b/cmd/minikube/cmd/cache.go index 50d57ca9aaf8..7933f4f7ec39 100644 --- a/cmd/minikube/cmd/cache.go +++ b/cmd/minikube/cmd/cache.go @@ -50,6 +50,7 @@ var addCacheCmd = &cobra.Command{ Short: "Add an image to local cache.", Long: "Add an image to local cache.", Run: func(cmd *cobra.Command, args []string) { + out.WarningT("\"minikube cache\" will be deprecated in upcoming versions, please switch to \"minikube image load\"") // Cache and load images into docker daemon if err := machine.CacheAndLoadImages(args, cacheAddProfiles()); err != nil { exit.Error(reason.InternalCacheLoad, "Failed to cache and load images", err)