diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index fb7747b34ad7..c4de4264a67f 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "os" + "strconv" "strings" "text/template" "time" @@ -80,8 +81,9 @@ const ( Paused = 418 // I'm a teapot! // 5xx signifies a server-side error (that may be retryable) - Error = 500 - Unknown = 520 + Error = 500 + InsufficientStorage = 507 + Unknown = 520 ) var ( @@ -100,8 +102,13 @@ var ( 418: "Paused", 500: "Error", + 507: "InsufficientStorage", 520: "Unknown", } + + codeDetails = map[int]string{ + 507: "/var is almost out of disk space", + } ) // Status holds string representations of component states @@ -258,7 +265,6 @@ func exitCode(statuses []*Status) int { // nodeStatus looks up the status of a node func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*Status, error) { - controlPlane := n.ControlPlane name := driver.MachineName(cc, n) @@ -315,6 +321,17 @@ func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*St return st, err } + // Check storage + p, err := machine.DiskUsed(cr, "/var") + if err != nil { + glog.Errorf("failed to get storage capacity of /var: %v", err) + st.Host = state.Error.String() + return st, err + } + if p >= 99 { + st.Host = codeNames[InsufficientStorage] + } + stk := kverify.KubeletStatus(cr) glog.Infof("%s kubelet status = %s", name, stk) st.Kubelet = stk.String() @@ -423,12 +440,15 @@ func readEventLog(name string) ([]cloudevents.Event, time.Time, error) { // clusterState converts Status structs into a ClusterState struct func clusterState(sts []*Status) ClusterState { + sc := statusCode(sts[0].Host) cs := ClusterState{ BinaryVersion: version.GetVersion(), BaseState: BaseState{ - Name: ClusterFlagValue(), - StatusCode: statusCode(sts[0].APIServer), + Name: ClusterFlagValue(), + StatusCode: sc, + StatusName: sts[0].Host, + StatusDetail: codeDetails[sc], }, Components: map[string]BaseState{ @@ -499,6 +519,26 @@ func clusterState(sts []*Status) ClusterState { finalStep = data glog.Infof("transient code %d (%q) for step: %+v", transientCode, codeNames[transientCode], data) } + if ev.Type() == "io.k8s.sigs.minikube.error" { + var data map[string]string + err := ev.DataAs(&data) + if err != nil { + glog.Errorf("unable to parse data: %v\nraw data: %s", err, ev.Data()) + continue + } + exitCode, err := strconv.Atoi(data["exitcode"]) + if err != nil { + glog.Errorf("unable to convert exit code to int: %v", err) + continue + } + transientCode = exitCode + for _, n := range cs.Nodes { + n.StatusCode = transientCode + n.StatusName = codeNames[n.StatusCode] + } + + glog.Infof("transient code %d (%q) for step: %+v", transientCode, codeNames[transientCode], data) + } } if finalStep != nil { @@ -514,6 +554,7 @@ func clusterState(sts []*Status) ClusterState { } cs.StatusName = codeNames[cs.StatusCode] + cs.StatusDetail = codeDetails[cs.StatusCode] return cs } diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index c7104d2673c0..748a11908622 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -73,6 +73,8 @@ const ( MinikubeActivePodmanEnv = "MINIKUBE_ACTIVE_PODMAN" // MinikubeForceSystemdEnv is used to force systemd as cgroup manager for the container runtime MinikubeForceSystemdEnv = "MINIKUBE_FORCE_SYSTEMD" + // TestDiskUsedEnv is used in integration tests for insufficient storage with 'minikube status' + TestDiskUsedEnv = "MINIKUBE_TEST_STORAGE_CAPACITY" ) var ( diff --git a/pkg/minikube/exit/exit.go b/pkg/minikube/exit/exit.go index b0a575700faa..dce7ce1eb513 100644 --- a/pkg/minikube/exit/exit.go +++ b/pkg/minikube/exit/exit.go @@ -29,16 +29,17 @@ import ( // Exit codes based on sysexits(3) const ( - Failure = 1 // Failure represents a general failure code - Interrupted = 2 // Ctrl-C (SIGINT) - BadUsage = 64 // Usage represents an incorrect command line - Data = 65 // Data represents incorrect data supplied by the user - NoInput = 66 // NoInput represents that the input file did not exist or was not readable - Unavailable = 69 // Unavailable represents when a service was unavailable - Software = 70 // Software represents an internal software error. - IO = 74 // IO represents an I/O error - Config = 78 // Config represents an unconfigured or misconfigured state - Permissions = 77 // Permissions represents a permissions error + Failure = 1 // Failure represents a general failure code + Interrupted = 2 // Ctrl-C (SIGINT) + BadUsage = 64 // Usage represents an incorrect command line + Data = 65 // Data represents incorrect data supplied by the user + NoInput = 66 // NoInput represents that the input file did not exist or was not readable + Unavailable = 69 // Unavailable represents when a service was unavailable + Software = 70 // Software represents an internal software error. + IO = 74 // IO represents an I/O error + Permissions = 77 // Permissions represents a permissions error + Config = 78 // Config represents an unconfigured or misconfigured state + InsufficientStorage = 507 // InsufficientStorage represents insufficient storage in the VM/container ) // UsageT outputs a templated usage error and exits with error code 64 diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index eb8437e9377e..5328f92ece53 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" "net" + "os" "os/exec" "path" "path/filepath" @@ -212,24 +213,37 @@ func postStartValidations(h *host.Host, drvName string) { if !driver.IsKIC(drvName) { return } - // make sure /var isn't full, otherwise warn - output, err := h.RunSSHCommand("df -h /var | awk 'NR==2{print $5}'") + r, err := CommandRunner(h) if err != nil { - glog.Warningf("error running df -h /var: %v", err) + glog.Warningf("error getting command runner: %v", err) } - output = strings.TrimSpace(output) - output = strings.Trim(output, "%") - percentageFull, err := strconv.Atoi(output) + // make sure /var isn't full, otherwise warn + percentageFull, err := DiskUsed(r, "/var") if err != nil { glog.Warningf("error getting percentage of /var that is free: %v", err) } if percentageFull >= 99 { - exit.WithError("", fmt.Errorf("docker daemon out of memory. No space left on device")) + exit.WithCodeT(exit.InsufficientStorage, "docker daemon out of memory. No space left on device") } if percentageFull > 80 { - out.WarningT("The docker daemon is almost out of memory, run 'docker system prune' to free up space") + out.ErrT(out.Tip, "The docker daemon is almost out of memory, run 'docker system prune' to free up space") + } +} + +// DiskUsed returns the capacity of dir in the VM/container as a percentage +func DiskUsed(cr command.Runner, dir string) (int, error) { + if s := os.Getenv(constants.TestDiskUsedEnv); s != "" { + return strconv.Atoi(s) + } + output, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf("df -h %s | awk 'NR==2{print $5}'", dir))) + if err != nil { + glog.Warningf("error running df -h /var: %v\n%v", err, output.Output()) + return 0, err } + percentage := strings.TrimSpace(output.Stdout.String()) + percentage = strings.Trim(percentage, "%") + return strconv.Atoi(percentage) } // postStart are functions shared between startHost and fixHost diff --git a/pkg/minikube/out/register/cloud_events.go b/pkg/minikube/out/register/cloud_events.go index 5ad88effca92..d654e4ba78fd 100644 --- a/pkg/minikube/out/register/cloud_events.go +++ b/pkg/minikube/out/register/cloud_events.go @@ -97,7 +97,7 @@ func printAndRecordCloudEvent(log Log, data map[string]string) { fmt.Fprintln(outputFile, string(bs)) if eventFile != nil { - go storeEvent(bs) + storeEvent(bs) } } diff --git a/test/integration/status_test.go b/test/integration/status_test.go new file mode 100644 index 000000000000..15b8a702f899 --- /dev/null +++ b/test/integration/status_test.go @@ -0,0 +1,98 @@ +// +build integration + +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path" + "testing" + + "k8s.io/minikube/cmd/minikube/cmd" + "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/localpath" +) + +func TestInsufficientStorage(t *testing.T) { + if !KicDriver() { + t.Skip("only runs with docker driver") + } + profile := UniqueProfileName("insufficient-storage") + ctx, cancel := context.WithTimeout(context.Background(), Minutes(5)) + defer Cleanup(t, profile, cancel) + + startArgs := []string{"start", "-p", profile, "--output=json", "--wait=true"} + startArgs = append(startArgs, StartArgs()...) + c := exec.CommandContext(ctx, Target(), startArgs...) + // artificially set /var to 100% capacity + c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestDiskUsedEnv)) + + rr, err := Run(t, c) + if err == nil { + t.Fatalf("expected command to fail, but it succeeded: %v\n%v", rr.Command(), err) + } + + // make sure 'minikube status' has correct output + stdout := runStatusCmd(ctx, t, profile) + verifyClusterState(t, stdout) + + // try deleting events.json and make sure this still works + eventsFile := path.Join(localpath.MiniPath(), "profiles", profile, "events.json") + if err := os.Remove(eventsFile); err != nil { + t.Fatalf("removing %s", eventsFile) + } + stdout = runStatusCmd(ctx, t, profile) + verifyClusterState(t, stdout) +} + +// runStatusCmd runs the status command and returns stdout +func runStatusCmd(ctx context.Context, t *testing.T, profile string) []byte { + // make sure minikube status shows insufficient storage + c := exec.CommandContext(ctx, Target(), "status", "-p", profile, "--output=json", "--layout=cluster") + // artificially set /var to 100% capacity + c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestDiskUsedEnv)) + rr, err := Run(t, c) + // status exits non-0 if status isn't Running + if err == nil { + t.Fatalf("expected command to fail, but it succeeded: %v\n%v", rr.Command(), err) + } + return rr.Stdout.Bytes() +} + +func verifyClusterState(t *testing.T, contents []byte) { + var cs cmd.ClusterState + if err := json.Unmarshal(contents, &cs); err != nil { + t.Fatalf("unmarshalling: %v", err) + } + // verify the status looks as we expect + if cs.StatusCode != cmd.InsufficientStorage { + t.Fatalf("incorrect status code: %v", cs.StatusCode) + } + if cs.StatusName != "InsufficientStorage" { + t.Fatalf("incorrect status name: %v", cs.StatusName) + } + for _, n := range cs.Nodes { + if n.StatusCode != cmd.InsufficientStorage { + t.Fatalf("incorrect node status code: %v", cs.StatusCode) + } + } +}