minikube status should display InsufficientStorage status

cmd/minikube/cmd/status.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strconv"
 	"strings"
 	"text/template"
 	"time"
@@ -80,8 +81,9 @@ const (
 	Paused   = 418 // I'm a teapot!
 
 	// 5xx signifies a server-side error (that may be retryable)
-	Error   = 500
-	Unknown = 520
+	Error               = 500
+	InsufficientStorage = 507
+	Unknown             = 520
 )
 
 var (
@@ -100,8 +102,13 @@ var (
 		418: "Paused",
 
 		500: "Error",
+		507: "InsufficientStorage",
 		520: "Unknown",
 	}
+
+	codeDetails = map[int]string{
+		507: "/var is almost out of disk space",
+	}
 )
 
 // Status holds string representations of component states
@@ -258,7 +265,6 @@ func exitCode(statuses []*Status) int {
 
 // nodeStatus looks up the status of a node
 func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*Status, error) {
-
 	controlPlane := n.ControlPlane
 	name := driver.MachineName(cc, n)
 
@@ -315,6 +321,17 @@ func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*St
 		return st, err
 	}
 
+	// Check storage
+	p, err := machine.MemoryCapacity(cr, "/var")
+	if err != nil {
+		glog.Errorf("failed to get memory capacity of /var: %v", err)
+		st.Host = state.Error.String()
+		return st, err
+	}
+	if p >= 99 {
+		st.Host = codeNames[InsufficientStorage]
+	}
+
 	stk := kverify.KubeletStatus(cr)
 	glog.Infof("%s kubelet status = %s", name, stk)
 	st.Kubelet = stk.String()
@@ -412,7 +429,9 @@ func readEventLog(name string) ([]cloudevents.Event, time.Time, error) {
 	scanner := bufio.NewScanner(f)
 	for scanner.Scan() {
 		ev := cloudevents.NewEvent()
-		if err = json.Unmarshal(scanner.Bytes(), &ev); err != nil {
+		b := scanner.Bytes()
+		if err = json.Unmarshal(b, &ev); err != nil {
+			glog.Infof("error unmarshalling data: %v\n%v", err, string(b))
 			return events, st.ModTime(), err
 		}
 		events = append(events, ev)
@@ -423,12 +442,16 @@ func readEventLog(name string) ([]cloudevents.Event, time.Time, error) {
 
 // clusterState converts Status structs into a ClusterState struct
 func clusterState(sts []*Status) ClusterState {
+	sc := statusCode(sts[0].Host)
+	glog.Infof("Initial status code is %v", sc)
 	cs := ClusterState{
 		BinaryVersion: version.GetVersion(),
 
 		BaseState: BaseState{
-			Name:       ClusterFlagValue(),
-			StatusCode: statusCode(sts[0].APIServer),
+			Name:         ClusterFlagValue(),
+			StatusCode:   sc,
+			StatusName:   sts[0].Host,
+			StatusDetail: codeDetails[sc],
 		},
 
 		Components: map[string]BaseState{
@@ -466,6 +489,8 @@ func clusterState(sts []*Status) ClusterState {
 		glog.Errorf("unable to read event log: %v", err)
 		return cs
 	}
+	glog.Infof("Found %v events in event log", len(evs))
+	glog.Infof("%v", evs)
 
 	transientCode := 0
 	var finalStep map[string]string
@@ -479,6 +504,7 @@ func clusterState(sts []*Status) ClusterState {
 				glog.Errorf("unable to parse data: %v\nraw data: %s", err, ev.Data())
 				continue
 			}
+			glog.Infof("looking at step %v", data["name"])
 
 			switch data["name"] {
 			case string(register.InitialSetup):
@@ -499,6 +525,28 @@ func clusterState(sts []*Status) ClusterState {
 			finalStep = data
 			glog.Infof("transient code %d (%q) for step: %+v", transientCode, codeNames[transientCode], data)
 		}
+		if ev.Type() == "io.k8s.sigs.minikube.error" {
+			var data map[string]string
+			err := ev.DataAs(&data)
+			if err != nil {
+				glog.Errorf("unable to parse data: %v\nraw data: %s", err, ev.Data())
+				continue
+			}
+			glog.Infof("exit code is %v", data["exitcode"])
+			exitCode, err := strconv.Atoi(strings.Trim(data["exitcode"], "\n"))
+			if err != nil {
+				glog.Errorf("unable to convert exit code to int: %v", err)
+				glog.Error("(event) ", data)
+				continue
+			}
+			transientCode = exitCode
+			for _, n := range cs.Nodes {
+				n.StatusCode = transientCode
+				n.StatusName = codeNames[n.StatusCode]
+			}
+
+			glog.Infof("transient code %d (%q) for step: %+v", transientCode, codeNames[transientCode], data)
+		}
 	}
 
 	if finalStep != nil {
@@ -508,12 +556,14 @@ func clusterState(sts []*Status) ClusterState {
 			cs.Step = strings.TrimSpace(finalStep["name"])
 			cs.StepDetail = strings.TrimSpace(finalStep["message"])
 			if transientCode != 0 {
+				glog.Infof("resetting status code to transient code %v", transientCode)
 				cs.StatusCode = transientCode
 			}
 		}
 	}
 
 	cs.StatusName = codeNames[cs.StatusCode]
+	cs.StatusDetail = codeDetails[cs.StatusCode]
 	return cs
 }
 

cmd/minikube/cmd/test

@@ -0,0 +1,14 @@
+{"data":{"currentstep":"0","message":"[insufficient-storage-20200820141526-290506] minikube v1.12.3 on Debian 9.13\n","name":"Initial Minikube Setup","totalsteps":"12"},"datacontenttype":"application/json","id":"3eb59878-f133-442e-9b72-6d732f3352e2","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"}
+{"data":{"message":"KUBECONFIG=/home/jenkins/minikube-integration/linux-amd64-docker-9034-287957-2c74cf22f910c2fc2ef59a5e206db9f3d3d72b82/kubeconfig\n"},"datacontenttype":"application/json","id":"a6150a88-f9db-41b9-8d21-e9f1504e2098","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"}
+{"data":{"message":"MINIKUBE_BIN=out/minikube-linux-amd64\n"},"datacontenttype":"application/json","id":"8453e611-1d14-4431-bd19-6692b144498b","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"}
+{"data":{"message":"MINIKUBE_HOME=/home/jenkins/minikube-integration/linux-amd64-docker-9034-287957-2c74cf22f910c2fc2ef59a5e206db9f3d3d72b82/.minikube\n"},"datacontenttype":"application/json","id":"0e2b6d5d-197a-4dd3-9505-dd81d151d5f0","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"}
+{"data":{"message":"MINIKUBE_LOCATION=9034\n"},"datacontenttype":"application/json","id":"db23721d-481c-4500-9765-5a0229227b68","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"}
+{"data":{"message":"MINIKUBE_TEST_MEMORY_CAPACITY=100\n"},"datacontenttype":"application/json","id":"9dd5de29-0153-4113-acf7-d7969bdafbbe","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.info"}
+{"data":{"currentstep":"1","message":"Using the docker driver based on user configuration\n","name":"Selecting Driver","totalsteps":"12"},"datacontenttype":"application/json","id":"46709583-0f6a-4987-adc8-b5692148998b","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"}
+{"data":{"message":"\n"},"datacontenttype":"application/json","id":"3c38872d-eb51-489d-adf7-4b0db614587c","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"}
+{"data":{"message":"'docker' driver reported a issue that could affect the performance.\n"},"datacontenttype":"application/json","id":"555e511f-a750-41ab-8886-d231c859ed81","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.warning"}
+{"data":{"message":"Suggestion: enable overlayfs kernel module on your Linux\n"},"datacontenttype":"application/json","id":"c9e6237c-cd54-4d7c-888d-1839304c3af0","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"}
+{"data":{"message":"\n"},"datacontenttype":"application/json","id":"2024d4e9-4c93-4dc9-a050-971ada6dfc8a","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"}
+{"data":{"currentstep":"3","message":"Starting control plane node insufficient-storage-20200820141526-290506 in cluster insufficient-storage-20200820141526-290506\n","name":"Starting Node","totalsteps":"12"},"datacontenttype":"application/json","id":"9a6c0793-4eea-4002-aa81-b28a111b4e9c","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"}
+{"data":{"currentstep":"6","message":"Creating docker container (CPUs=2, Memory=7500MB) ...\n","name":"Creating Container","totalsteps":"12"},"datacontenttype":"application/json","id":"3a2ad48f-d4ec-4b69-b2d2-f1d86f4c5cc9","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"}
+{"data":{"exitcode":"507","message":"docker daemon out of memory. No space left on device\n"},"datacontenttype":"application/json","id":"deb577e1-e4ce-40e0-bf2c-7157a1f569ad","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"}

pkg/minikube/constants/constants.go

@@ -72,6 +72,8 @@ const (
 	MinikubeActivePodmanEnv = "MINIKUBE_ACTIVE_PODMAN"
 	// MinikubeForceSystemdEnv is used to force systemd as cgroup manager for the container runtime
 	MinikubeForceSystemdEnv = "MINIKUBE_FORCE_SYSTEMD"
+	// TestMemoryCapacityEnv is used in integration tests for insufficient storage with 'minikube status'
+	TestMemoryCapacityEnv = "MINIKUBE_TEST_MEMORY_CAPACITY"
 )
 
 var (

pkg/minikube/exit/exit.go

@@ -29,16 +29,17 @@ import (
 
 // Exit codes based on sysexits(3)
 const (
-	Failure     = 1  // Failure represents a general failure code
-	Interrupted = 2  // Ctrl-C (SIGINT)
-	BadUsage    = 64 // Usage represents an incorrect command line
-	Data        = 65 // Data represents incorrect data supplied by the user
-	NoInput     = 66 // NoInput represents that the input file did not exist or was not readable
-	Unavailable = 69 // Unavailable represents when a service was unavailable
-	Software    = 70 // Software represents an internal software error.
-	IO          = 74 // IO represents an I/O error
-	Config      = 78 // Config represents an unconfigured or misconfigured state
-	Permissions = 77 // Permissions represents a permissions error
+	Failure             = 1   // Failure represents a general failure code
+	Interrupted         = 2   // Ctrl-C (SIGINT)
+	BadUsage            = 64  // Usage represents an incorrect command line
+	Data                = 65  // Data represents incorrect data supplied by the user
+	NoInput             = 66  // NoInput represents that the input file did not exist or was not readable
+	Unavailable         = 69  // Unavailable represents when a service was unavailable
+	Software            = 70  // Software represents an internal software error.
+	IO                  = 74  // IO represents an I/O error
+	Permissions         = 77  // Permissions represents a permissions error
+	Config              = 78  // Config represents an unconfigured or misconfigured state
+	InsufficientStorage = 507 // InsufficientStorage represents insufficient storage in the VM/container
 )
 
 // UsageT outputs a templated usage error and exits with error code 64

pkg/minikube/machine/start.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
@@ -212,23 +213,38 @@ func postStartValidations(h *host.Host, drvName string) {
 	if !driver.IsKIC(drvName) {
 		return
 	}
-	// make sure /var isn't full, otherwise warn
-	output, err := h.RunSSHCommand("df -h /var | awk 'NR==2{print $5}'")
+	r, err := CommandRunner(h)
 	if err != nil {
-		glog.Warningf("error running df -h /var: %v", err)
+		glog.Warningf("error getting command runner: %v", err)
 	}
-	output = strings.Trim(output, "\n")
-	percentageFull, err := strconv.Atoi(output[:len(output)-1])
+	// make sure /var isn't full, otherwise warn
+	percentageFull, err := MemoryCapacity(r, "/var")
 	if err != nil {
 		glog.Warningf("error getting percentage of /var that is free: %v", err)
 	}
 	if percentageFull >= 99 {
-		exit.WithError("", fmt.Errorf("docker daemon out of memory. No space left on device"))
+		exit.WithCodeT(exit.InsufficientStorage, "docker daemon out of memory. No space left on device")
 	}
 
 	if percentageFull > 80 {
-		out.WarningT("The docker daemon is almost out of memory, run 'docker system prune' to free up space")
+		out.ErrT(out.Tip, "The docker daemon is almost out of memory, run 'docker system prune' to free up space")
+	}
+}
+
+// MemoryCapacity returns the capacity of dir in the VM/container
+func MemoryCapacity(cr command.Runner, dir string) (int, error) {
+	if s := os.Getenv(constants.TestMemoryCapacityEnv); s != "" {
+		return strconv.Atoi(s)
+	}
+	output, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf("df -h %s | awk 'NR==2{print $5}'", dir)))
+	if err != nil {
+		glog.Warningf("error running df -h /var: %v", err)
+		fmt.Println(output.Output())
+		return 0, errors.Wrap(err, "CRY")
 	}
+	percentage := output.Stdout.String()
+	percentage = strings.Trim(percentage, "\n")
+	return strconv.Atoi(percentage[:len(percentage)-1])
 }
 
 // postStart are functions shared between startHost and fixHost

pkg/minikube/out/register/cloud_events.go

@@ -97,7 +97,7 @@ func printAndRecordCloudEvent(log Log, data map[string]string) {
 	fmt.Fprintln(outputFile, string(bs))
 
 	if eventFile != nil {
-		go storeEvent(bs)
+		storeEvent(bs)
 	}
 }
 

test/integration/status_test.go

@@ -0,0 +1,98 @@
+// +build integration
+
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"testing"
+
+	"k8s.io/minikube/cmd/minikube/cmd"
+	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/localpath"
+)
+
+func TestInsufficientStorage(t *testing.T) {
+	if !KicDriver() {
+		t.Skip("only runs with docker driver")
+	}
+	profile := UniqueProfileName("insufficient-storage")
+	ctx, cancel := context.WithTimeout(context.Background(), Minutes(5))
+	defer Cleanup(t, profile, cancel)
+
+	startArgs := []string{"start", "-p", profile, "--output=json", "--wait=true"}
+	startArgs = append(startArgs, StartArgs()...)
+	c := exec.CommandContext(ctx, Target(), startArgs...)
+	// artificially set /var to 100% capacity
+	c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestMemoryCapacityEnv))
+
+	rr, err := Run(t, c)
+	if err == nil {
+		t.Fatalf("expected command to fail, but it succeeded: %v\n%v", rr.Command(), err)
+	}
+
+	// make sure 'minikube status' has correct output
+	stdout := runStatusCmd(ctx, t, profile)
+	verifyClusterState(t, stdout)
+
+	// try deleting events.json and make sure this still works
+	eventsFile := path.Join(localpath.MiniPath(), "profiles", profile, "events.json")
+	if err := os.Remove(eventsFile); err != nil {
+		t.Fatalf("removing %s", eventsFile)
+	}
+	stdout = runStatusCmd(ctx, t, profile)
+	verifyClusterState(t, stdout)
+}
+
+// runStatusCmd runs the status command and returns stdout
+func runStatusCmd(ctx context.Context, t *testing.T, profile string) []byte {
+	// make sure minikube status shows insufficient storage
+	c := exec.CommandContext(ctx, Target(), "status", "-p", profile, "--output=json", "--layout=cluster", "--alsologtostderr")
+	// artificially set /var to 100% capacity
+	c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestMemoryCapacityEnv))
+	rr, err := Run(t, c)
+	// status exits non-0 if status isn't Running
+	if err == nil {
+		t.Fatalf("expected command to fail, but it succeeded: %v\n%v", rr.Command(), err)
+	}
+	return rr.Stdout.Bytes()
+}
+
+func verifyClusterState(t *testing.T, contents []byte) {
+	var cs cmd.ClusterState
+	if err := json.Unmarshal(contents, &cs); err != nil {
+		t.Fatalf("unmarshalling: %v", err)
+	}
+	// verify the status looks as we expect
+	if cs.StatusCode != cmd.InsufficientStorage {
+		t.Fatalf("incorrect status code: %v", cs.StatusCode)
+	}
+	if cs.StatusName != "InsufficientStorage" {
+		t.Fatalf("incorrect status name: %v", cs.StatusName)
+	}
+	for _, n := range cs.Nodes {
+		if n.StatusCode != cmd.InsufficientStorage {
+			t.Fatalf("incorrect node status code: %v", cs.StatusCode)
+		}
+	}
+}