From 3dba9745351df0908e6fc309f43a344bf8b7c688 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 19 Aug 2020 13:25:48 -0400 Subject: [PATCH 01/14] Make sure 'minikube status' displays InsufficientStorage for cluster and nodes if there is insufficient storage. Output now looks like: ``` { "Name": "minikube", "StatusCode": 507, "StatusName": "InsufficientStorage", "StatusDetail": "/var is almost out of disk space", "Step": "Starting Node", "StepDetail": "Updating the running docker \"minikube\" container ...", "BinaryVersion": "v1.12.3", "Components": { "kubeconfig": { "Name": "kubeconfig", "StatusCode": 500, "StatusName": "" } }, "Nodes": [ { "Name": "minikube", "StatusCode": 507, "StatusName": "InsufficientStorage", "Components": { "apiserver": { "Name": "apiserver", "StatusCode": 405, "StatusName": "Stopped" }, "kubelet": { "Name": "kubelet", "StatusCode": 405, "StatusName": "Stopped" } } } ] } ``` --- cmd/minikube/cmd/status.go | 45 ++++++++++++++++++++++++++++--- go.mod | 1 - pkg/minikube/machine/start.go | 27 ++++++++++++++----- pkg/minikube/out/errors/errors.go | 24 +++++++++++++++++ 4 files changed, 86 insertions(+), 11 deletions(-) create mode 100644 pkg/minikube/out/errors/errors.go diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index fb7747b34ad7..4ffbf35c8897 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -43,6 +43,7 @@ import ( "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/mustload" "k8s.io/minikube/pkg/minikube/node" + mkerrors "k8s.io/minikube/pkg/minikube/out/errors" "k8s.io/minikube/pkg/minikube/out/register" "k8s.io/minikube/pkg/version" ) @@ -80,8 +81,9 @@ const ( Paused = 418 // I'm a teapot! // 5xx signifies a server-side error (that may be retryable) - Error = 500 - Unknown = 520 + Error = 500 + InsufficientStorage = 507 + Unknown = 520 ) var ( @@ -100,8 +102,13 @@ var ( 418: "Paused", 500: "Error", + 507: "InsufficientStorage", 520: "Unknown", } + + codeDetails = map[int]string{ + 507: "/var is almost out of disk space", + } ) // Status holds string representations of component states @@ -258,7 +265,6 @@ func exitCode(statuses []*Status) int { // nodeStatus looks up the status of a node func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*Status, error) { - controlPlane := n.ControlPlane name := driver.MachineName(cc, n) @@ -315,6 +321,17 @@ func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*St return st, err } + // Check storage + p, err := machine.MemoryCapacity(cr, "/var") + if err != nil { + glog.Errorf("failed to get memory capacity of /var: %v", err) + st.Host = state.Error.String() + return st, err + } + if p >= 99 { + st.Host = codeNames[InsufficientStorage] + } + stk := kverify.KubeletStatus(cr) glog.Infof("%s kubelet status = %s", name, stk) st.Kubelet = stk.String() @@ -499,6 +516,27 @@ func clusterState(sts []*Status) ClusterState { finalStep = data glog.Infof("transient code %d (%q) for step: %+v", transientCode, codeNames[transientCode], data) } + if ev.Type() == "io.k8s.sigs.minikube.error" { + var data map[string]string + err := ev.DataAs(&data) + if err != nil { + glog.Errorf("unable to parse data: %v\nraw data: %s", err, ev.Data()) + continue + } + msg := strings.Trim(data["message"], "\n") + switch msg { + case mkerrors.ErrDockerOOM.Error(): + transientCode = InsufficientStorage + default: + continue + } + for _, n := range cs.Nodes { + n.StatusCode = transientCode + n.StatusName = codeNames[n.StatusCode] + } + + glog.Infof("transient code %d (%q) for step: %+v", transientCode, codeNames[transientCode], data) + } } if finalStep != nil { @@ -514,6 +552,7 @@ func clusterState(sts []*Status) ClusterState { } cs.StatusName = codeNames[cs.StatusCode] + cs.StatusDetail = codeDetails[cs.StatusCode] return cs } diff --git a/go.mod b/go.mod index 51b5d1276604..78f91ae7da2b 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,6 @@ require ( github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097 golang.org/x/build v0.0.0-20190927031335-2835ba2e683f golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 - golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 // indirect golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a golang.org/x/sys v0.0.0-20200523222454-059865788121 diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index b4ee40cccaf2..41ccc9a0ffac 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -42,6 +42,7 @@ import ( "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/out" + mkerrors "k8s.io/minikube/pkg/minikube/out/errors" "k8s.io/minikube/pkg/minikube/out/register" "k8s.io/minikube/pkg/minikube/proxy" "k8s.io/minikube/pkg/minikube/registry" @@ -211,23 +212,35 @@ func postStartValidations(h *host.Host, drvName string) { if !driver.IsKIC(drvName) { return } - // make sure /var isn't full, otherwise warn - output, err := h.RunSSHCommand("df -h /var | awk 'NR==2{print $5}'") + r, err := CommandRunner(h) if err != nil { - glog.Warningf("error running df -h /var: %v", err) + glog.Warningf("error getting command runner: %v", err) } - output = strings.Trim(output, "\n") - percentageFull, err := strconv.Atoi(output[:len(output)-1]) + // make sure /var isn't full, otherwise warn + percentageFull, err := MemoryCapacity(r, "/var") if err != nil { glog.Warningf("error getting percentage of /var that is free: %v", err) } if percentageFull >= 99 { - exit.WithError("", fmt.Errorf("docker daemon out of memory. No space left on device")) + exit.WithError("", mkerrors.ErrDockerOOM) } if percentageFull > 80 { - out.WarningT("The docker daemon is almost out of memory, run 'docker system prune' to free up space") + out.Err("The docker daemon is almost out of memory, run 'docker system prune' to free up space") + } +} + +// MemoryCapacity returns the capacity of dir in the VM/container +func MemoryCapacity(cr command.Runner, dir string) (int, error) { + output, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf("df -h %s | awk 'NR==2{print $5}'", dir))) + if err != nil { + glog.Warningf("error running df -h /var: %v", err) + fmt.Println(output.Output()) + return 0, errors.Wrap(err, "CRY") } + percentage := string(output.Stdout.Bytes()) + percentage = strings.Trim(percentage, "\n") + return strconv.Atoi(percentage[:len(percentage)-1]) } // postStart are functions shared between startHost and fixHost diff --git a/pkg/minikube/out/errors/errors.go b/pkg/minikube/out/errors/errors.go new file mode 100644 index 000000000000..8932f7c06b32 --- /dev/null +++ b/pkg/minikube/out/errors/errors.go @@ -0,0 +1,24 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package errors + +import "fmt" + +var ( + // ErrDockerOOM is the error returned when the docker daemon is out of memory + ErrDockerOOM = fmt.Errorf("docker daemon out of memory. No space left on device") +) From 1783cc2939ec058c2742293693a39c45d4f626a4 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 19 Aug 2020 13:32:36 -0400 Subject: [PATCH 02/14] Make sure minikube status shows status name and details --- cmd/minikube/cmd/status.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index 4ffbf35c8897..40a09d051da3 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -440,12 +440,15 @@ func readEventLog(name string) ([]cloudevents.Event, time.Time, error) { // clusterState converts Status structs into a ClusterState struct func clusterState(sts []*Status) ClusterState { + sc := statusCode(sts[0].Host) cs := ClusterState{ BinaryVersion: version.GetVersion(), BaseState: BaseState{ - Name: ClusterFlagValue(), - StatusCode: statusCode(sts[0].APIServer), + Name: ClusterFlagValue(), + StatusCode: sc, + StatusName: sts[0].Host, + StatusDetail: codeDetails[sc], }, Components: map[string]BaseState{ From 4d1c1c0dc3e8d2cf4f8f15ac405b5d110dc5a669 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 19 Aug 2020 14:16:00 -0400 Subject: [PATCH 03/14] Add integration test for insufficient storage Make sure that if /var has full storage, the output of `minikube status` is correct regardless of whether events.json is present or not --- pkg/minikube/constants/constants.go | 2 + pkg/minikube/machine/start.go | 6 +- test/integration/status_test.go | 96 +++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 test/integration/status_test.go diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 19d2b73ecdd8..a044c0c39794 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -72,6 +72,8 @@ const ( MinikubeActivePodmanEnv = "MINIKUBE_ACTIVE_PODMAN" // MinikubeForceSystemdEnv is used to force systemd as cgroup manager for the container runtime MinikubeForceSystemdEnv = "MINIKUBE_FORCE_SYSTEMD" + // TestMemoryCapacityEnv is used in integration tests for insufficient storage with 'minikube status' + TestMemoryCapacityEnv = "MINIKUBE_TEST_MEMORY_CAPACITY" ) var ( diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 41ccc9a0ffac..922dc6d56ab1 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" "net" + "os" "os/exec" "path" "path/filepath" @@ -232,13 +233,16 @@ func postStartValidations(h *host.Host, drvName string) { // MemoryCapacity returns the capacity of dir in the VM/container func MemoryCapacity(cr command.Runner, dir string) (int, error) { + if s := os.Getenv(constants.TestMemoryCapacityEnv); s != "" { + return strconv.Atoi(s) + } output, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf("df -h %s | awk 'NR==2{print $5}'", dir))) if err != nil { glog.Warningf("error running df -h /var: %v", err) fmt.Println(output.Output()) return 0, errors.Wrap(err, "CRY") } - percentage := string(output.Stdout.Bytes()) + percentage := output.Stdout.String() percentage = strings.Trim(percentage, "\n") return strconv.Atoi(percentage[:len(percentage)-1]) } diff --git a/test/integration/status_test.go b/test/integration/status_test.go new file mode 100644 index 000000000000..7879acefa2d2 --- /dev/null +++ b/test/integration/status_test.go @@ -0,0 +1,96 @@ +// +build integration + +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path" + "testing" + + "k8s.io/minikube/cmd/minikube/cmd" + "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/localpath" +) + +func TestInsufficientStorage(t *testing.T) { + if !KicDriver() { + t.Skip("only runs with docker driver") + } + profile := UniqueProfileName("insufficient-storage") + ctx, cancel := context.WithTimeout(context.Background(), Minutes(5)) + defer Cleanup(t, profile, cancel) + + startArgs := []string{"start", "-p", profile, "--output=json", "--wait=true"} + startArgs = append(startArgs, StartArgs()...) + c := exec.CommandContext(ctx, Target(), startArgs...) + // artificially set /var to 100% capacity + c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestMemoryCapacityEnv)) + + rr, err := Run(t, c) + if err == nil { + t.Fatalf("expected command to fail, but it succeeded: %v\n%v", rr.Command(), err) + } + stdout := runStatusCmd(t, ctx, profile) + verifyClusterState(t, stdout) + + // try deleting events.json and make sure this still works + eventsFile := path.Join(localpath.MiniPath(), "profiles", profile, "events.json") + if err := os.Remove(eventsFile); err != nil { + t.Fatalf("removing %s", eventsFile) + } + stdout = runStatusCmd(t, ctx, profile) + verifyClusterState(t, stdout) +} + +// runStatusCmd runs the status command and returns stdout +func runStatusCmd(t *testing.T, ctx context.Context, profile string) []byte { + // make sure minikube status shows insufficient storage + c := exec.CommandContext(ctx, Target(), "status", "-p", profile, "--output=json", "--layout=cluster") + // artificially set /var to 100% capacity + c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestMemoryCapacityEnv)) + rr, err := Run(t, c) + // status exits non-0 if status isn't Running + if err == nil { + t.Fatalf("expected command to fail, but it succeeded: %v\n%v", rr.Command(), err) + } + return rr.Stdout.Bytes() +} + +func verifyClusterState(t *testing.T, contents []byte) { + var cs cmd.ClusterState + if err := json.Unmarshal(contents, &cs); err != nil { + t.Fatalf("unmarshalling: %v", err) + } + // verify the status looks as we expect + if cs.StatusCode != cmd.InsufficientStorage { + t.Fatalf("incorrect status code: %v", cs.StatusCode) + } + if cs.StatusName != "InsufficientStorage" { + t.Fatalf("incorrect status name: %v", cs.StatusName) + } + for _, n := range cs.Nodes { + if n.StatusCode != cmd.InsufficientStorage { + t.Fatalf("incorrect node status code: %v", cs.StatusCode) + } + } +} From 399dad0884b3e4c922c9bf338388963e83cf76d5 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 19 Aug 2020 15:15:39 -0400 Subject: [PATCH 04/14] fix lint --- test/integration/status_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/status_test.go b/test/integration/status_test.go index 7879acefa2d2..c6f5d92d81f6 100644 --- a/test/integration/status_test.go +++ b/test/integration/status_test.go @@ -50,7 +50,7 @@ func TestInsufficientStorage(t *testing.T) { if err == nil { t.Fatalf("expected command to fail, but it succeeded: %v\n%v", rr.Command(), err) } - stdout := runStatusCmd(t, ctx, profile) + stdout := runStatusCmd(ctx, t, profile) verifyClusterState(t, stdout) // try deleting events.json and make sure this still works @@ -58,12 +58,12 @@ func TestInsufficientStorage(t *testing.T) { if err := os.Remove(eventsFile); err != nil { t.Fatalf("removing %s", eventsFile) } - stdout = runStatusCmd(t, ctx, profile) + stdout = runStatusCmd(ctx, t, profile) verifyClusterState(t, stdout) } // runStatusCmd runs the status command and returns stdout -func runStatusCmd(t *testing.T, ctx context.Context, profile string) []byte { +func runStatusCmd(ctx context.Context, t *testing.T, profile string) []byte { // make sure minikube status shows insufficient storage c := exec.CommandContext(ctx, Target(), "status", "-p", profile, "--output=json", "--layout=cluster") // artificially set /var to 100% capacity From e1deec7019256712008ff48b629397a7f007f764 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 19 Aug 2020 17:57:56 -0400 Subject: [PATCH 05/14] improve output --- pkg/minikube/machine/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 8f5cf5925ce6..33c6221363dc 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -228,7 +228,7 @@ func postStartValidations(h *host.Host, drvName string) { } if percentageFull > 80 { - out.Err("The docker daemon is almost out of memory, run 'docker system prune' to free up space") + out.ErrT(out.Tip, "The docker daemon is almost out of memory, run 'docker system prune' to free up space") } } From 3cb90267702f6241cf7bcea550bb8d052bdf1043 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 20 Aug 2020 13:18:15 -0400 Subject: [PATCH 06/14] validate last event is the correct error --- go.mod | 1 + go.sum | 9 +++++++++ test/integration/status_test.go | 10 ++++++++++ 3 files changed, 20 insertions(+) diff --git a/go.mod b/go.mod index 78f91ae7da2b..40cd6c8c90c9 100644 --- a/go.mod +++ b/go.mod @@ -86,6 +86,7 @@ require ( k8s.io/kubectl v0.0.0 k8s.io/kubernetes v1.17.3 k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab // indirect + sigs.k8s.io/kind v0.6.0 // indirect sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible // indirect sigs.k8s.io/sig-storage-lib-external-provisioner/v5 v5.0.0 ) diff --git a/go.sum b/go.sum index 8e67a76cd02e..2cd32f3c3c0b 100644 --- a/go.sum +++ b/go.sum @@ -132,6 +132,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZq github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053 h1:H/GMMKYPkEIC3DF/JWQz8Pdd+Feifov2EIgGfNpeogI= +github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053/go.mod h1:xW8sBma2LE3QxFSzCnH9qe6gAE2yO9GvQaWwX89HxbE= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ= @@ -714,6 +716,7 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -1208,7 +1211,9 @@ golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1435,6 +1440,8 @@ gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20191106092431-e228e37189d3 h1:ho4SukHOmqjp7XHH7nPNx7GcgDK6ObVflhAQAT7MvpE= +gopkg.in/yaml.v3 v3.0.0-20191106092431-e228e37189d3/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= @@ -1511,6 +1518,8 @@ pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/kind v0.6.0 h1:D1ToBztLzDJhqJADTO+oX4DC8DwqqjNm026y+g38pT0= +sigs.k8s.io/kind v0.6.0/go.mod h1:dhW5h0O4PRVq8B6eExphIa9mBnrlBzxEz3R/P1YcYj0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible h1:qV3eFdgCp7Cp/ORjkJI9VBBEOntT+z385jLqdBtmgHA= sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible/go.mod h1:qhqLyNwJC49PoUalmtzYb4s9fT8HOMBTLbTY1QoVOqI= diff --git a/test/integration/status_test.go b/test/integration/status_test.go index c6f5d92d81f6..a315f18bb05c 100644 --- a/test/integration/status_test.go +++ b/test/integration/status_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/minikube/cmd/minikube/cmd" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/out/errors" ) func TestInsufficientStorage(t *testing.T) { @@ -50,6 +51,15 @@ func TestInsufficientStorage(t *testing.T) { if err == nil { t.Fatalf("expected command to fail, but it succeeded: %v\n%v", rr.Command(), err) } + // make sure last event is the error we expect + ces, err := cloudEvents(t, rr) + if err != nil { + t.Fatalf("error marshalling into cloud events: %v", err) + } + last := ces[len(ces)-1] + last.validateData(t, "message", errors.ErrDockerOOM.Error()) + + // make sure 'minikube status' has correct output stdout := runStatusCmd(ctx, t, profile) verifyClusterState(t, stdout) From 83eba00d47ec41fe32a951f72fd7aac098ac5e5b Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 20 Aug 2020 14:42:55 -0400 Subject: [PATCH 07/14] add deubgging --- cmd/minikube/cmd/status.go | 4 ++++ test/integration/status_test.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index 40a09d051da3..5727bed10185 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -441,6 +441,7 @@ func readEventLog(name string) ([]cloudevents.Event, time.Time, error) { // clusterState converts Status structs into a ClusterState struct func clusterState(sts []*Status) ClusterState { sc := statusCode(sts[0].Host) + glog.Infof("Initial status code is %v", sc) cs := ClusterState{ BinaryVersion: version.GetVersion(), @@ -499,6 +500,7 @@ func clusterState(sts []*Status) ClusterState { glog.Errorf("unable to parse data: %v\nraw data: %s", err, ev.Data()) continue } + glog.Infof("looking at step %v", data["name"]) switch data["name"] { case string(register.InitialSetup): @@ -531,6 +533,7 @@ func clusterState(sts []*Status) ClusterState { case mkerrors.ErrDockerOOM.Error(): transientCode = InsufficientStorage default: + glog.Infof("skipping error we don't have info about") continue } for _, n := range cs.Nodes { @@ -549,6 +552,7 @@ func clusterState(sts []*Status) ClusterState { cs.Step = strings.TrimSpace(finalStep["name"]) cs.StepDetail = strings.TrimSpace(finalStep["message"]) if transientCode != 0 { + glog.Infof("resetting status code to transient code %v", transientCode) cs.StatusCode = transientCode } } diff --git a/test/integration/status_test.go b/test/integration/status_test.go index a315f18bb05c..cbc6414da5a9 100644 --- a/test/integration/status_test.go +++ b/test/integration/status_test.go @@ -75,7 +75,7 @@ func TestInsufficientStorage(t *testing.T) { // runStatusCmd runs the status command and returns stdout func runStatusCmd(ctx context.Context, t *testing.T, profile string) []byte { // make sure minikube status shows insufficient storage - c := exec.CommandContext(ctx, Target(), "status", "-p", profile, "--output=json", "--layout=cluster") + c := exec.CommandContext(ctx, Target(), "status", "-p", profile, "--output=json", "--layout=cluster", "--alsologtostderr") // artificially set /var to 100% capacity c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestMemoryCapacityEnv)) rr, err := Run(t, c) From 7eda3a77115cb82a06a6ddb73feb44522b031b83 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 20 Aug 2020 15:27:06 -0400 Subject: [PATCH 08/14] Create new exit code for InsufficientStorage --- cmd/minikube/cmd/status.go | 12 +++++------- pkg/minikube/exit/exit.go | 21 +++++++++++---------- pkg/minikube/machine/start.go | 3 +-- pkg/minikube/out/errors/errors.go | 24 ------------------------ test/integration/status_test.go | 8 -------- 5 files changed, 17 insertions(+), 51 deletions(-) delete mode 100644 pkg/minikube/out/errors/errors.go diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index 5727bed10185..e301d72df98b 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "os" + "strconv" "strings" "text/template" "time" @@ -43,7 +44,6 @@ import ( "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/mustload" "k8s.io/minikube/pkg/minikube/node" - mkerrors "k8s.io/minikube/pkg/minikube/out/errors" "k8s.io/minikube/pkg/minikube/out/register" "k8s.io/minikube/pkg/version" ) @@ -528,14 +528,12 @@ func clusterState(sts []*Status) ClusterState { glog.Errorf("unable to parse data: %v\nraw data: %s", err, ev.Data()) continue } - msg := strings.Trim(data["message"], "\n") - switch msg { - case mkerrors.ErrDockerOOM.Error(): - transientCode = InsufficientStorage - default: - glog.Infof("skipping error we don't have info about") + exitCode, err := strconv.Atoi(data["exitcode"]) + if err != nil { + glog.Errorf("unable to convert exit code to int: %v", err) continue } + transientCode = exitCode for _, n := range cs.Nodes { n.StatusCode = transientCode n.StatusName = codeNames[n.StatusCode] diff --git a/pkg/minikube/exit/exit.go b/pkg/minikube/exit/exit.go index b0a575700faa..dce7ce1eb513 100644 --- a/pkg/minikube/exit/exit.go +++ b/pkg/minikube/exit/exit.go @@ -29,16 +29,17 @@ import ( // Exit codes based on sysexits(3) const ( - Failure = 1 // Failure represents a general failure code - Interrupted = 2 // Ctrl-C (SIGINT) - BadUsage = 64 // Usage represents an incorrect command line - Data = 65 // Data represents incorrect data supplied by the user - NoInput = 66 // NoInput represents that the input file did not exist or was not readable - Unavailable = 69 // Unavailable represents when a service was unavailable - Software = 70 // Software represents an internal software error. - IO = 74 // IO represents an I/O error - Config = 78 // Config represents an unconfigured or misconfigured state - Permissions = 77 // Permissions represents a permissions error + Failure = 1 // Failure represents a general failure code + Interrupted = 2 // Ctrl-C (SIGINT) + BadUsage = 64 // Usage represents an incorrect command line + Data = 65 // Data represents incorrect data supplied by the user + NoInput = 66 // NoInput represents that the input file did not exist or was not readable + Unavailable = 69 // Unavailable represents when a service was unavailable + Software = 70 // Software represents an internal software error. + IO = 74 // IO represents an I/O error + Permissions = 77 // Permissions represents a permissions error + Config = 78 // Config represents an unconfigured or misconfigured state + InsufficientStorage = 507 // InsufficientStorage represents insufficient storage in the VM/container ) // UsageT outputs a templated usage error and exits with error code 64 diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 33c6221363dc..a07e29a70a0d 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -43,7 +43,6 @@ import ( "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/out" - mkerrors "k8s.io/minikube/pkg/minikube/out/errors" "k8s.io/minikube/pkg/minikube/out/register" "k8s.io/minikube/pkg/minikube/proxy" "k8s.io/minikube/pkg/minikube/registry" @@ -224,7 +223,7 @@ func postStartValidations(h *host.Host, drvName string) { glog.Warningf("error getting percentage of /var that is free: %v", err) } if percentageFull >= 99 { - exit.WithError("", mkerrors.ErrDockerOOM) + exit.WithCodeT(exit.InsufficientStorage, "docker daemon out of memory. No space left on device") } if percentageFull > 80 { diff --git a/pkg/minikube/out/errors/errors.go b/pkg/minikube/out/errors/errors.go deleted file mode 100644 index 8932f7c06b32..000000000000 --- a/pkg/minikube/out/errors/errors.go +++ /dev/null @@ -1,24 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package errors - -import "fmt" - -var ( - // ErrDockerOOM is the error returned when the docker daemon is out of memory - ErrDockerOOM = fmt.Errorf("docker daemon out of memory. No space left on device") -) diff --git a/test/integration/status_test.go b/test/integration/status_test.go index cbc6414da5a9..3527e1266aa5 100644 --- a/test/integration/status_test.go +++ b/test/integration/status_test.go @@ -30,7 +30,6 @@ import ( "k8s.io/minikube/cmd/minikube/cmd" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/localpath" - "k8s.io/minikube/pkg/minikube/out/errors" ) func TestInsufficientStorage(t *testing.T) { @@ -51,13 +50,6 @@ func TestInsufficientStorage(t *testing.T) { if err == nil { t.Fatalf("expected command to fail, but it succeeded: %v\n%v", rr.Command(), err) } - // make sure last event is the error we expect - ces, err := cloudEvents(t, rr) - if err != nil { - t.Fatalf("error marshalling into cloud events: %v", err) - } - last := ces[len(ces)-1] - last.validateData(t, "message", errors.ErrDockerOOM.Error()) // make sure 'minikube status' has correct output stdout := runStatusCmd(ctx, t, profile) From 482b2bf624a1e57301e83775bcaa39e29f779971 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 20 Aug 2020 16:45:24 -0400 Subject: [PATCH 09/14] additional debugging --- cmd/minikube/cmd/status.go | 10 ++++++++-- cmd/minikube/cmd/test | 14 ++++++++++++++ go.mod | 1 - go.sum | 9 --------- 4 files changed, 22 insertions(+), 12 deletions(-) create mode 100644 cmd/minikube/cmd/test diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index e301d72df98b..ed849a0a0ac9 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -429,7 +429,9 @@ func readEventLog(name string) ([]cloudevents.Event, time.Time, error) { scanner := bufio.NewScanner(f) for scanner.Scan() { ev := cloudevents.NewEvent() - if err = json.Unmarshal(scanner.Bytes(), &ev); err != nil { + b := scanner.Bytes() + if err = json.Unmarshal(b, &ev); err != nil { + glog.Infof("error unmarshalling data: %v\n%v", err, string(b)) return events, st.ModTime(), err } events = append(events, ev) @@ -487,6 +489,8 @@ func clusterState(sts []*Status) ClusterState { glog.Errorf("unable to read event log: %v", err) return cs } + glog.Infof("Found %v events in event log", len(evs)) + glog.Infof("%v", evs) transientCode := 0 var finalStep map[string]string @@ -528,9 +532,11 @@ func clusterState(sts []*Status) ClusterState { glog.Errorf("unable to parse data: %v\nraw data: %s", err, ev.Data()) continue } - exitCode, err := strconv.Atoi(data["exitcode"]) + glog.Infof("exit code is %v", data["exitcode"]) + exitCode, err := strconv.Atoi(strings.Trim(data["exitcode"], "\n")) if err != nil { glog.Errorf("unable to convert exit code to int: %v", err) + glog.Error("(event) ", data) continue } transientCode = exitCode diff --git a/cmd/minikube/cmd/test b/cmd/minikube/cmd/test new file mode 100644 index 000000000000..2e8a767c400e --- /dev/null +++ b/cmd/minikube/cmd/test @@ -0,0 +1,14 @@ +{"data":{"currentstep":"0","message":"[insufficient-storage-20200820141526-290506] minikube v1.12.3 on Debian 9.13\n","name":"Initial Minikube Setup","totalsteps":"12"},"datacontenttype":"application/json","id":"3eb59878-f133-442e-9b72-6d732f3352e2","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"} +{"data":{"message":"KUBECONFIG=/home/jenkins/minikube-integration/linux-amd64-docker-9034-287957-2c74cf22f910c2fc2ef59a5e206db9f3d3d72b82/kubeconfig\n"},"datacontenttype":"application/json","id":"a6150a88-f9db-41b9-8d21-e9f1504e2098","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"} +{"data":{"message":"MINIKUBE_BIN=out/minikube-linux-amd64\n"},"datacontenttype":"application/json","id":"8453e611-1d14-4431-bd19-6692b144498b","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"} +{"data":{"message":"MINIKUBE_HOME=/home/jenkins/minikube-integration/linux-amd64-docker-9034-287957-2c74cf22f910c2fc2ef59a5e206db9f3d3d72b82/.minikube\n"},"datacontenttype":"application/json","id":"0e2b6d5d-197a-4dd3-9505-dd81d151d5f0","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"} +{"data":{"message":"MINIKUBE_LOCATION=9034\n"},"datacontenttype":"application/json","id":"db23721d-481c-4500-9765-5a0229227b68","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"} +{"data":{"message":"MINIKUBE_TEST_MEMORY_CAPACITY=100\n"},"datacontenttype":"application/json","id":"9dd5de29-0153-4113-acf7-d7969bdafbbe","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"} +{"data":{"currentstep":"1","message":"Using the docker driver based on user configuration\n","name":"Selecting Driver","totalsteps":"12"},"datacontenttype":"application/json","id":"46709583-0f6a-4987-adc8-b5692148998b","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"} +{"data":{"message":"\n"},"datacontenttype":"application/json","id":"3c38872d-eb51-489d-adf7-4b0db614587c","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"} +{"data":{"message":"'docker' driver reported a issue that could affect the performance.\n"},"datacontenttype":"application/json","id":"555e511f-a750-41ab-8886-d231c859ed81","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.warning"} +{"data":{"message":"Suggestion: enable overlayfs kernel module on your Linux\n"},"datacontenttype":"application/json","id":"c9e6237c-cd54-4d7c-888d-1839304c3af0","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"} +{"data":{"message":"\n"},"datacontenttype":"application/json","id":"2024d4e9-4c93-4dc9-a050-971ada6dfc8a","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"} +{"data":{"currentstep":"3","message":"Starting control plane node insufficient-storage-20200820141526-290506 in cluster insufficient-storage-20200820141526-290506\n","name":"Starting Node","totalsteps":"12"},"datacontenttype":"application/json","id":"9a6c0793-4eea-4002-aa81-b28a111b4e9c","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"} +{"data":{"currentstep":"6","message":"Creating docker container (CPUs=2, Memory=7500MB) ...\n","name":"Creating Container","totalsteps":"12"},"datacontenttype":"application/json","id":"3a2ad48f-d4ec-4b69-b2d2-f1d86f4c5cc9","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"} +{"data":{"exitcode":"507","message":"docker daemon out of memory. No space left on device\n"},"datacontenttype":"application/json","id":"deb577e1-e4ce-40e0-bf2c-7157a1f569ad","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"} \ No newline at end of file diff --git a/go.mod b/go.mod index 40cd6c8c90c9..78f91ae7da2b 100644 --- a/go.mod +++ b/go.mod @@ -86,7 +86,6 @@ require ( k8s.io/kubectl v0.0.0 k8s.io/kubernetes v1.17.3 k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab // indirect - sigs.k8s.io/kind v0.6.0 // indirect sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible // indirect sigs.k8s.io/sig-storage-lib-external-provisioner/v5 v5.0.0 ) diff --git a/go.sum b/go.sum index 2cd32f3c3c0b..8e67a76cd02e 100644 --- a/go.sum +++ b/go.sum @@ -132,8 +132,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZq github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053 h1:H/GMMKYPkEIC3DF/JWQz8Pdd+Feifov2EIgGfNpeogI= -github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053/go.mod h1:xW8sBma2LE3QxFSzCnH9qe6gAE2yO9GvQaWwX89HxbE= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ= @@ -716,7 +714,6 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -1211,9 +1208,7 @@ golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1440,8 +1435,6 @@ gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20191106092431-e228e37189d3 h1:ho4SukHOmqjp7XHH7nPNx7GcgDK6ObVflhAQAT7MvpE= -gopkg.in/yaml.v3 v3.0.0-20191106092431-e228e37189d3/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= @@ -1518,8 +1511,6 @@ pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/kind v0.6.0 h1:D1ToBztLzDJhqJADTO+oX4DC8DwqqjNm026y+g38pT0= -sigs.k8s.io/kind v0.6.0/go.mod h1:dhW5h0O4PRVq8B6eExphIa9mBnrlBzxEz3R/P1YcYj0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible h1:qV3eFdgCp7Cp/ORjkJI9VBBEOntT+z385jLqdBtmgHA= sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible/go.mod h1:qhqLyNwJC49PoUalmtzYb4s9fT8HOMBTLbTY1QoVOqI= From 4fcffca3db413f80c832b8b4878b35c8050be333 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 20 Aug 2020 20:48:18 -0400 Subject: [PATCH 10/14] write events to file, but not in parallel --- pkg/minikube/out/register/cloud_events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/out/register/cloud_events.go b/pkg/minikube/out/register/cloud_events.go index 5ad88effca92..d654e4ba78fd 100644 --- a/pkg/minikube/out/register/cloud_events.go +++ b/pkg/minikube/out/register/cloud_events.go @@ -97,7 +97,7 @@ func printAndRecordCloudEvent(log Log, data map[string]string) { fmt.Fprintln(outputFile, string(bs)) if eventFile != nil { - go storeEvent(bs) + storeEvent(bs) } } From 44b93f4776891a703f29b309f80ff9a18765b051 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 21 Aug 2020 09:49:48 -0400 Subject: [PATCH 11/14] Change MemoryCapacity to StorageCapacity to be more accurate --- cmd/minikube/cmd/status.go | 2 +- cmd/minikube/cmd/test | 14 -------------- pkg/minikube/constants/constants.go | 4 ++-- pkg/minikube/machine/start.go | 8 ++++---- test/integration/status_test.go | 4 ++-- 5 files changed, 9 insertions(+), 23 deletions(-) delete mode 100644 cmd/minikube/cmd/test diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index ed849a0a0ac9..fc931be6df85 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -322,7 +322,7 @@ func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*St } // Check storage - p, err := machine.MemoryCapacity(cr, "/var") + p, err := machine.StorageCapacity(cr, "/var") if err != nil { glog.Errorf("failed to get memory capacity of /var: %v", err) st.Host = state.Error.String() diff --git a/cmd/minikube/cmd/test b/cmd/minikube/cmd/test deleted file mode 100644 index 2e8a767c400e..000000000000 --- a/cmd/minikube/cmd/test +++ /dev/null @@ -1,14 +0,0 @@ -{"data":{"currentstep":"0","message":"[insufficient-storage-20200820141526-290506] minikube v1.12.3 on Debian 9.13\n","name":"Initial Minikube Setup","totalsteps":"12"},"datacontenttype":"application/json","id":"3eb59878-f133-442e-9b72-6d732f3352e2","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"} -{"data":{"message":"KUBECONFIG=/home/jenkins/minikube-integration/linux-amd64-docker-9034-287957-2c74cf22f910c2fc2ef59a5e206db9f3d3d72b82/kubeconfig\n"},"datacontenttype":"application/json","id":"a6150a88-f9db-41b9-8d21-e9f1504e2098","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"} -{"data":{"message":"MINIKUBE_BIN=out/minikube-linux-amd64\n"},"datacontenttype":"application/json","id":"8453e611-1d14-4431-bd19-6692b144498b","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"} -{"data":{"message":"MINIKUBE_HOME=/home/jenkins/minikube-integration/linux-amd64-docker-9034-287957-2c74cf22f910c2fc2ef59a5e206db9f3d3d72b82/.minikube\n"},"datacontenttype":"application/json","id":"0e2b6d5d-197a-4dd3-9505-dd81d151d5f0","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"} -{"data":{"message":"MINIKUBE_LOCATION=9034\n"},"datacontenttype":"application/json","id":"db23721d-481c-4500-9765-5a0229227b68","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"} -{"data":{"message":"MINIKUBE_TEST_MEMORY_CAPACITY=100\n"},"datacontenttype":"application/json","id":"9dd5de29-0153-4113-acf7-d7969bdafbbe","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"} -{"data":{"currentstep":"1","message":"Using the docker driver based on user configuration\n","name":"Selecting Driver","totalsteps":"12"},"datacontenttype":"application/json","id":"46709583-0f6a-4987-adc8-b5692148998b","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"} -{"data":{"message":"\n"},"datacontenttype":"application/json","id":"3c38872d-eb51-489d-adf7-4b0db614587c","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"} -{"data":{"message":"'docker' driver reported a issue that could affect the performance.\n"},"datacontenttype":"application/json","id":"555e511f-a750-41ab-8886-d231c859ed81","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.warning"} -{"data":{"message":"Suggestion: enable overlayfs kernel module on your Linux\n"},"datacontenttype":"application/json","id":"c9e6237c-cd54-4d7c-888d-1839304c3af0","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"} -{"data":{"message":"\n"},"datacontenttype":"application/json","id":"2024d4e9-4c93-4dc9-a050-971ada6dfc8a","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"} -{"data":{"currentstep":"3","message":"Starting control plane node insufficient-storage-20200820141526-290506 in cluster insufficient-storage-20200820141526-290506\n","name":"Starting Node","totalsteps":"12"},"datacontenttype":"application/json","id":"9a6c0793-4eea-4002-aa81-b28a111b4e9c","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"} -{"data":{"currentstep":"6","message":"Creating docker container (CPUs=2, Memory=7500MB) ...\n","name":"Creating Container","totalsteps":"12"},"datacontenttype":"application/json","id":"3a2ad48f-d4ec-4b69-b2d2-f1d86f4c5cc9","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"} -{"data":{"exitcode":"507","message":"docker daemon out of memory. No space left on device\n"},"datacontenttype":"application/json","id":"deb577e1-e4ce-40e0-bf2c-7157a1f569ad","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"} \ No newline at end of file diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index a044c0c39794..199f50760d98 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -72,8 +72,8 @@ const ( MinikubeActivePodmanEnv = "MINIKUBE_ACTIVE_PODMAN" // MinikubeForceSystemdEnv is used to force systemd as cgroup manager for the container runtime MinikubeForceSystemdEnv = "MINIKUBE_FORCE_SYSTEMD" - // TestMemoryCapacityEnv is used in integration tests for insufficient storage with 'minikube status' - TestMemoryCapacityEnv = "MINIKUBE_TEST_MEMORY_CAPACITY" + // TestStorageCapacityEnv is used in integration tests for insufficient storage with 'minikube status' + TestStorageCapacityEnv = "MINIKUBE_TEST_STORAGE_CAPACITY" ) var ( diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index a07e29a70a0d..502b033f9850 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -218,7 +218,7 @@ func postStartValidations(h *host.Host, drvName string) { glog.Warningf("error getting command runner: %v", err) } // make sure /var isn't full, otherwise warn - percentageFull, err := MemoryCapacity(r, "/var") + percentageFull, err := StorageCapacity(r, "/var") if err != nil { glog.Warningf("error getting percentage of /var that is free: %v", err) } @@ -231,9 +231,9 @@ func postStartValidations(h *host.Host, drvName string) { } } -// MemoryCapacity returns the capacity of dir in the VM/container -func MemoryCapacity(cr command.Runner, dir string) (int, error) { - if s := os.Getenv(constants.TestMemoryCapacityEnv); s != "" { +// StorageCapacity returns the capacity of dir in the VM/container as a percentage +func StorageCapacity(cr command.Runner, dir string) (int, error) { + if s := os.Getenv(constants.TestStorageCapacityEnv); s != "" { return strconv.Atoi(s) } output, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf("df -h %s | awk 'NR==2{print $5}'", dir))) diff --git a/test/integration/status_test.go b/test/integration/status_test.go index 3527e1266aa5..c89ed58762fa 100644 --- a/test/integration/status_test.go +++ b/test/integration/status_test.go @@ -44,7 +44,7 @@ func TestInsufficientStorage(t *testing.T) { startArgs = append(startArgs, StartArgs()...) c := exec.CommandContext(ctx, Target(), startArgs...) // artificially set /var to 100% capacity - c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestMemoryCapacityEnv)) + c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestStorageCapacityEnv)) rr, err := Run(t, c) if err == nil { @@ -69,7 +69,7 @@ func runStatusCmd(ctx context.Context, t *testing.T, profile string) []byte { // make sure minikube status shows insufficient storage c := exec.CommandContext(ctx, Target(), "status", "-p", profile, "--output=json", "--layout=cluster", "--alsologtostderr") // artificially set /var to 100% capacity - c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestMemoryCapacityEnv)) + c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestStorageCapacityEnv)) rr, err := Run(t, c) // status exits non-0 if status isn't Running if err == nil { From acd6589c5dc8323a2923fc9eb0f9a00153d70cc1 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 21 Aug 2020 09:53:13 -0400 Subject: [PATCH 12/14] Remove debugging --- cmd/minikube/cmd/status.go | 13 ++----------- pkg/minikube/machine/start.go | 5 ++--- test/integration/status_test.go | 2 +- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index fc931be6df85..228dbdb0e074 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -429,9 +429,7 @@ func readEventLog(name string) ([]cloudevents.Event, time.Time, error) { scanner := bufio.NewScanner(f) for scanner.Scan() { ev := cloudevents.NewEvent() - b := scanner.Bytes() - if err = json.Unmarshal(b, &ev); err != nil { - glog.Infof("error unmarshalling data: %v\n%v", err, string(b)) + if err = json.Unmarshal(scanner.Bytes(), &ev); err != nil { return events, st.ModTime(), err } events = append(events, ev) @@ -443,7 +441,6 @@ func readEventLog(name string) ([]cloudevents.Event, time.Time, error) { // clusterState converts Status structs into a ClusterState struct func clusterState(sts []*Status) ClusterState { sc := statusCode(sts[0].Host) - glog.Infof("Initial status code is %v", sc) cs := ClusterState{ BinaryVersion: version.GetVersion(), @@ -489,8 +486,6 @@ func clusterState(sts []*Status) ClusterState { glog.Errorf("unable to read event log: %v", err) return cs } - glog.Infof("Found %v events in event log", len(evs)) - glog.Infof("%v", evs) transientCode := 0 var finalStep map[string]string @@ -504,7 +499,6 @@ func clusterState(sts []*Status) ClusterState { glog.Errorf("unable to parse data: %v\nraw data: %s", err, ev.Data()) continue } - glog.Infof("looking at step %v", data["name"]) switch data["name"] { case string(register.InitialSetup): @@ -532,11 +526,9 @@ func clusterState(sts []*Status) ClusterState { glog.Errorf("unable to parse data: %v\nraw data: %s", err, ev.Data()) continue } - glog.Infof("exit code is %v", data["exitcode"]) - exitCode, err := strconv.Atoi(strings.Trim(data["exitcode"], "\n")) + exitCode, err := strconv.Atoi(data["exitcode"]) if err != nil { glog.Errorf("unable to convert exit code to int: %v", err) - glog.Error("(event) ", data) continue } transientCode = exitCode @@ -556,7 +548,6 @@ func clusterState(sts []*Status) ClusterState { cs.Step = strings.TrimSpace(finalStep["name"]) cs.StepDetail = strings.TrimSpace(finalStep["message"]) if transientCode != 0 { - glog.Infof("resetting status code to transient code %v", transientCode) cs.StatusCode = transientCode } } diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 502b033f9850..b170abe03dc1 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -238,9 +238,8 @@ func StorageCapacity(cr command.Runner, dir string) (int, error) { } output, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf("df -h %s | awk 'NR==2{print $5}'", dir))) if err != nil { - glog.Warningf("error running df -h /var: %v", err) - fmt.Println(output.Output()) - return 0, errors.Wrap(err, "CRY") + glog.Warningf("error running df -h /var: %v\n%v", err, output.Output()) + return 0, err } percentage := output.Stdout.String() percentage = strings.Trim(percentage, "\n") diff --git a/test/integration/status_test.go b/test/integration/status_test.go index c89ed58762fa..72ba6d5059be 100644 --- a/test/integration/status_test.go +++ b/test/integration/status_test.go @@ -67,7 +67,7 @@ func TestInsufficientStorage(t *testing.T) { // runStatusCmd runs the status command and returns stdout func runStatusCmd(ctx context.Context, t *testing.T, profile string) []byte { // make sure minikube status shows insufficient storage - c := exec.CommandContext(ctx, Target(), "status", "-p", profile, "--output=json", "--layout=cluster", "--alsologtostderr") + c := exec.CommandContext(ctx, Target(), "status", "-p", profile, "--output=json", "--layout=cluster") // artificially set /var to 100% capacity c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestStorageCapacityEnv)) rr, err := Run(t, c) From 97acf408e05c32a11d789da014a19f1d612dc2d4 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 21 Aug 2020 13:34:06 -0400 Subject: [PATCH 13/14] fix error wording --- cmd/minikube/cmd/status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index 228dbdb0e074..5547d4f2275d 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -324,7 +324,7 @@ func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*St // Check storage p, err := machine.StorageCapacity(cr, "/var") if err != nil { - glog.Errorf("failed to get memory capacity of /var: %v", err) + glog.Errorf("failed to get storage capacity of /var: %v", err) st.Host = state.Error.String() return st, err } From 88aece40d2e8c673a6f2b7d4f01c6ce6c104cda5 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 21 Aug 2020 15:05:37 -0400 Subject: [PATCH 14/14] change name --- cmd/minikube/cmd/status.go | 2 +- pkg/minikube/constants/constants.go | 4 ++-- pkg/minikube/machine/start.go | 8 ++++---- test/integration/status_test.go | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index 5547d4f2275d..c4de4264a67f 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -322,7 +322,7 @@ func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*St } // Check storage - p, err := machine.StorageCapacity(cr, "/var") + p, err := machine.DiskUsed(cr, "/var") if err != nil { glog.Errorf("failed to get storage capacity of /var: %v", err) st.Host = state.Error.String() diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 199f50760d98..3d9e0128f5c9 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -72,8 +72,8 @@ const ( MinikubeActivePodmanEnv = "MINIKUBE_ACTIVE_PODMAN" // MinikubeForceSystemdEnv is used to force systemd as cgroup manager for the container runtime MinikubeForceSystemdEnv = "MINIKUBE_FORCE_SYSTEMD" - // TestStorageCapacityEnv is used in integration tests for insufficient storage with 'minikube status' - TestStorageCapacityEnv = "MINIKUBE_TEST_STORAGE_CAPACITY" + // TestDiskUsedEnv is used in integration tests for insufficient storage with 'minikube status' + TestDiskUsedEnv = "MINIKUBE_TEST_STORAGE_CAPACITY" ) var ( diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index b170abe03dc1..867074e4ec34 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -218,7 +218,7 @@ func postStartValidations(h *host.Host, drvName string) { glog.Warningf("error getting command runner: %v", err) } // make sure /var isn't full, otherwise warn - percentageFull, err := StorageCapacity(r, "/var") + percentageFull, err := DiskUsed(r, "/var") if err != nil { glog.Warningf("error getting percentage of /var that is free: %v", err) } @@ -231,9 +231,9 @@ func postStartValidations(h *host.Host, drvName string) { } } -// StorageCapacity returns the capacity of dir in the VM/container as a percentage -func StorageCapacity(cr command.Runner, dir string) (int, error) { - if s := os.Getenv(constants.TestStorageCapacityEnv); s != "" { +// DiskUsed returns the capacity of dir in the VM/container as a percentage +func DiskUsed(cr command.Runner, dir string) (int, error) { + if s := os.Getenv(constants.TestDiskUsedEnv); s != "" { return strconv.Atoi(s) } output, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf("df -h %s | awk 'NR==2{print $5}'", dir))) diff --git a/test/integration/status_test.go b/test/integration/status_test.go index 72ba6d5059be..15b8a702f899 100644 --- a/test/integration/status_test.go +++ b/test/integration/status_test.go @@ -44,7 +44,7 @@ func TestInsufficientStorage(t *testing.T) { startArgs = append(startArgs, StartArgs()...) c := exec.CommandContext(ctx, Target(), startArgs...) // artificially set /var to 100% capacity - c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestStorageCapacityEnv)) + c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestDiskUsedEnv)) rr, err := Run(t, c) if err == nil { @@ -69,7 +69,7 @@ func runStatusCmd(ctx context.Context, t *testing.T, profile string) []byte { // make sure minikube status shows insufficient storage c := exec.CommandContext(ctx, Target(), "status", "-p", profile, "--output=json", "--layout=cluster") // artificially set /var to 100% capacity - c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestStorageCapacityEnv)) + c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestDiskUsedEnv)) rr, err := Run(t, c) // status exits non-0 if status isn't Running if err == nil {