From c005cda301588d81f4aa13f2ddfe5b596077a3bb Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 24 Sep 2020 13:03:57 -0400 Subject: [PATCH 1/7] Print solution message when there is no space left on device during preload extraction. --- pkg/drivers/kic/kic.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 6c01be18ba73..16a8703d2133 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -37,6 +37,8 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/download" + "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/reason" "k8s.io/minikube/pkg/minikube/sysinit" "k8s.io/minikube/pkg/util/retry" ) @@ -134,6 +136,9 @@ func (d *Driver) Create() error { glog.Infof("Starting extracting preloaded images to volume ...") // Extract preloaded images to container if err := oci.ExtractTarballToVolume(d.NodeConfig.OCIBinary, download.TarballPath(d.NodeConfig.KubernetesVersion, d.NodeConfig.ContainerRuntime), params.Name, d.NodeConfig.ImageDigest); err != nil { + if strings.Contains(err.Error(), "No space left on device") { + exit.Message(reason.RsrcInsufficientDockerStorage, "preload extraction failed: \"No space left on device\"") + } glog.Infof("Unable to extract preloaded tarball to volume: %v", err) } else { glog.Infof("duration metric: took %f seconds to extract preloaded images to volume", time.Since(t).Seconds()) From 3ee1eb029ccffe85854f78e6092ab41a7aee808f Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 24 Sep 2020 14:09:46 -0400 Subject: [PATCH 2/7] Update test to use new exit code for insufficient storage --- cmd/minikube/cmd/status.go | 21 +++++++++++---------- test/integration/status_test.go | 3 ++- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index df7184c0c1cb..e4f1c2eacfda 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -67,6 +67,8 @@ const ( Irrelevant = "Irrelevant" // New status modes, based roughly on HTTP/SMTP standards + InsufficientStorage = reason.ExInsufficientStorage + // 1xx signifies a transitional state. If retried, it will soon return a 2xx, 4xx, or 5xx Starting = 100 Pausing = 101 @@ -84,18 +86,18 @@ const ( Paused = 418 // I'm a teapot! // 5xx signifies a server-side error (that may be retryable) - Error = 500 - InsufficientStorage = 507 - Unknown = 520 + Error = 500 + Unknown = 520 ) var ( codeNames = map[int]string{ - 100: "Starting", - 101: "Pausing", - 102: "Unpausing", - 110: "Stopping", - 103: "Deleting", + reason.ExInsufficientStorage: "InsufficientStorage", + 100: "Starting", + 101: "Pausing", + 102: "Unpausing", + 110: "Stopping", + 103: "Deleting", 200: "OK", 203: "Warning", @@ -105,12 +107,11 @@ var ( 418: "Paused", 500: "Error", - 507: "InsufficientStorage", 520: "Unknown", } codeDetails = map[int]string{ - 507: "/var is almost out of disk space", + reason.ExInsufficientStorage: "/var is almost out of disk space", } ) diff --git a/test/integration/status_test.go b/test/integration/status_test.go index 15b8a702f899..b78bc737d401 100644 --- a/test/integration/status_test.go +++ b/test/integration/status_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/minikube/cmd/minikube/cmd" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/reason" ) func TestInsufficientStorage(t *testing.T) { @@ -84,7 +85,7 @@ func verifyClusterState(t *testing.T, contents []byte) { t.Fatalf("unmarshalling: %v", err) } // verify the status looks as we expect - if cs.StatusCode != cmd.InsufficientStorage { + if cs.StatusCode != reason.ExInsufficientStorage { t.Fatalf("incorrect status code: %v", cs.StatusCode) } if cs.StatusName != "InsufficientStorage" { From 998a0f7058fd2442ece8f196ba16d171f2ba1c9c Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 24 Sep 2020 14:15:15 -0400 Subject: [PATCH 3/7] add profile in failing test --- test/integration/functional_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 52d188190aee..d0247596c907 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -328,7 +328,7 @@ func validateMinikubeKubectlDirectCall(ctx context.Context, t *testing.T, profil } defer os.Remove(dstfn) // clean up - kubectlArgs := []string{"get", "pods"} + kubectlArgs := []string{"get", "pods", "--context", profile} rr, err := Run(t, exec.CommandContext(ctx, dstfn, kubectlArgs...)) if err != nil { t.Fatalf("failed to run kubectl directl. args %q: %v", rr.Command(), err) From da4833d80afa38c9c86861f8f286e8c411b5ea9a Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 28 Sep 2020 13:31:09 -0400 Subject: [PATCH 4/7] revert functional test --- test/integration/functional_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index d0247596c907..52d188190aee 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -328,7 +328,7 @@ func validateMinikubeKubectlDirectCall(ctx context.Context, t *testing.T, profil } defer os.Remove(dstfn) // clean up - kubectlArgs := []string{"get", "pods", "--context", profile} + kubectlArgs := []string{"get", "pods"} rr, err := Run(t, exec.CommandContext(ctx, dstfn, kubectlArgs...)) if err != nil { t.Fatalf("failed to run kubectl directl. args %q: %v", rr.Command(), err) From db23825bfc98c309f8ebcb61e92ccb0924ceddb7 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 30 Sep 2020 15:17:58 -0400 Subject: [PATCH 5/7] Move exit.Message function so that we aren't exiting from a library --- cmd/minikube/cmd/start.go | 9 +++++++++ pkg/drivers/kic/kic.go | 8 ++++---- pkg/drivers/kic/oci/errors.go | 3 +++ pkg/minikube/node/start.go | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 10f9c8636f89..a729883cb372 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -197,6 +197,9 @@ func runStart(cmd *cobra.Command, args []string) { machine.MaybeDisplayAdvice(err, ds.Name) if specified { // If the user specified a driver, don't fallback to anything else + if errors.Cause(err) == oci.ErrInsufficientDockerStorage { + exit.Message(reason.RsrcInsufficientDockerStorage, "preload extraction failed: \"No space left on device\"") + } exit.Error(reason.GuestProvision, "error provisioning host", err) } else { success := false @@ -224,6 +227,9 @@ func runStart(cmd *cobra.Command, args []string) { } } if !success { + if errors.Cause(err) == oci.ErrInsufficientDockerStorage { + exit.Message(reason.RsrcInsufficientDockerStorage, "preload extraction failed: \"No space left on device\"") + } exit.Error(reason.GuestProvision, "error provisioning host", err) } } @@ -248,6 +254,9 @@ func runStart(cmd *cobra.Command, args []string) { stopProfile(existing.Name) starter, err = provisionWithDriver(cmd, ds, existing) if err != nil { + if errors.Cause(err) == oci.ErrInsufficientDockerStorage { + exit.Message(reason.RsrcInsufficientDockerStorage, "preload extraction failed: \"No space left on device\"") + } exit.Error(reason.GuestProvision, "error provisioning host", err) } } diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 16a8703d2133..b1407257c73b 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -37,8 +37,6 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/download" - "k8s.io/minikube/pkg/minikube/exit" - "k8s.io/minikube/pkg/minikube/reason" "k8s.io/minikube/pkg/minikube/sysinit" "k8s.io/minikube/pkg/util/retry" ) @@ -126,6 +124,7 @@ func (d *Driver) Create() error { var waitForPreload sync.WaitGroup waitForPreload.Add(1) + var pErr error go func() { defer waitForPreload.Done() // If preload doesn't exist, don't bother extracting tarball to volume @@ -137,7 +136,8 @@ func (d *Driver) Create() error { // Extract preloaded images to container if err := oci.ExtractTarballToVolume(d.NodeConfig.OCIBinary, download.TarballPath(d.NodeConfig.KubernetesVersion, d.NodeConfig.ContainerRuntime), params.Name, d.NodeConfig.ImageDigest); err != nil { if strings.Contains(err.Error(), "No space left on device") { - exit.Message(reason.RsrcInsufficientDockerStorage, "preload extraction failed: \"No space left on device\"") + pErr = oci.ErrInsufficientDockerStorage + return } glog.Infof("Unable to extract preloaded tarball to volume: %v", err) } else { @@ -154,7 +154,7 @@ func (d *Driver) Create() error { } waitForPreload.Wait() - return nil + return pErr } // prepareSSH will generate keys and copy to the container so minikube ssh works diff --git a/pkg/drivers/kic/oci/errors.go b/pkg/drivers/kic/oci/errors.go index 8c30201f978c..292ef601fbe0 100644 --- a/pkg/drivers/kic/oci/errors.go +++ b/pkg/drivers/kic/oci/errors.go @@ -45,6 +45,9 @@ var ErrExitedUnexpectedly = errors.New("container exited unexpectedly") // ErrDaemonInfo is thrown when docker/podman info is failing or not responding var ErrDaemonInfo = errors.New("daemon info not responding") +// ErrInsufficientDockerStorage is thrown when there is not more storage for docker +var ErrInsufficientDockerStorage = &FailFastError{errors.New("insufficient docker storage, no space left on device")} + // LogContainerDebug will print relevant docker/podman infos after a container fails func LogContainerDebug(ociBin string, name string) string { rr, err := containerInspect(ociBin, name) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 6982f1013514..036998b33ca6 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -383,7 +383,7 @@ func startHost(api libmachine.API, cc *config.ClusterConfig, n *config.Node, del } } - if _, ff := err.(*oci.FailFastError); ff { + if err, ff := errors.Cause(err).(*oci.FailFastError); ff { glog.Infof("will skip retrying to create machine because error is not retriable: %v", err) return host, exists, err } From 6538306d14cf78deab6a94d74c0953fbd1417ace Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 30 Sep 2020 15:31:44 -0400 Subject: [PATCH 6/7] move exit into separate function --- cmd/minikube/cmd/start.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index a729883cb372..45a27678c5db 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -197,10 +197,7 @@ func runStart(cmd *cobra.Command, args []string) { machine.MaybeDisplayAdvice(err, ds.Name) if specified { // If the user specified a driver, don't fallback to anything else - if errors.Cause(err) == oci.ErrInsufficientDockerStorage { - exit.Message(reason.RsrcInsufficientDockerStorage, "preload extraction failed: \"No space left on device\"") - } - exit.Error(reason.GuestProvision, "error provisioning host", err) + exitGuestProvision(err) } else { success := false // Walk down the rest of the options @@ -227,10 +224,7 @@ func runStart(cmd *cobra.Command, args []string) { } } if !success { - if errors.Cause(err) == oci.ErrInsufficientDockerStorage { - exit.Message(reason.RsrcInsufficientDockerStorage, "preload extraction failed: \"No space left on device\"") - } - exit.Error(reason.GuestProvision, "error provisioning host", err) + exitGuestProvision(err) } } } @@ -254,10 +248,7 @@ func runStart(cmd *cobra.Command, args []string) { stopProfile(existing.Name) starter, err = provisionWithDriver(cmd, ds, existing) if err != nil { - if errors.Cause(err) == oci.ErrInsufficientDockerStorage { - exit.Message(reason.RsrcInsufficientDockerStorage, "preload extraction failed: \"No space left on device\"") - } - exit.Error(reason.GuestProvision, "error provisioning host", err) + exitGuestProvision(err) } } } @@ -1272,3 +1263,10 @@ func exitIfNotForced(r reason.Kind, message string, v ...out.V) { } out.Error(r, message, v...) } + +func exitGuestProvision(err error) { + if errors.Cause(err) == oci.ErrInsufficientDockerStorage { + exit.Message(reason.RsrcInsufficientDockerStorage, "preload extraction failed: \"No space left on device\"") + } + exit.Error(reason.GuestProvision, "error provisioning host", err) +} From d6acf7f4cbfb77ca2f401bfe2565291299b1bf79 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 5 Oct 2020 11:49:28 -0400 Subject: [PATCH 7/7] fix issues --- pkg/drivers/kic/kic.go | 5 ++++- test/integration/status_test.go | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index f49bf1526aa9..8a896118b30b 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -157,6 +157,9 @@ func (d *Driver) Create() error { glog.Infof("duration metric: took %f seconds to extract preloaded images to volume", time.Since(t).Seconds()) } }() + if pErr == oci.ErrInsufficientDockerStorage { + return pErr + } if err := oci.CreateContainerNode(params); err != nil { return errors.Wrap(err, "create kic node") @@ -167,7 +170,7 @@ func (d *Driver) Create() error { } waitForPreload.Wait() - return pErr + return nil } // prepareSSH will generate keys and copy to the container so minikube ssh works diff --git a/test/integration/status_test.go b/test/integration/status_test.go index b78bc737d401..15b8a702f899 100644 --- a/test/integration/status_test.go +++ b/test/integration/status_test.go @@ -30,7 +30,6 @@ import ( "k8s.io/minikube/cmd/minikube/cmd" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/localpath" - "k8s.io/minikube/pkg/minikube/reason" ) func TestInsufficientStorage(t *testing.T) { @@ -85,7 +84,7 @@ func verifyClusterState(t *testing.T, contents []byte) { t.Fatalf("unmarshalling: %v", err) } // verify the status looks as we expect - if cs.StatusCode != reason.ExInsufficientStorage { + if cs.StatusCode != cmd.InsufficientStorage { t.Fatalf("incorrect status code: %v", cs.StatusCode) } if cs.StatusName != "InsufficientStorage" {