Skip to content

Commit

Permalink
Merge pull request #9034 from priyawadhwa/var-status-code
Browse files Browse the repository at this point in the history
`minikube status` should display InsufficientStorage status
  • Loading branch information
priyawadhwa authored Aug 24, 2020
2 parents 59af2c1 + a767b64 commit 7c84e2e
Show file tree
Hide file tree
Showing 6 changed files with 180 additions and 24 deletions.
51 changes: 46 additions & 5 deletions cmd/minikube/cmd/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"fmt"
"io"
"os"
"strconv"
"strings"
"text/template"
"time"
Expand Down Expand Up @@ -80,8 +81,9 @@ const (
Paused = 418 // I'm a teapot!

// 5xx signifies a server-side error (that may be retryable)
Error = 500
Unknown = 520
Error = 500
InsufficientStorage = 507
Unknown = 520
)

var (
Expand All @@ -100,8 +102,13 @@ var (
418: "Paused",

500: "Error",
507: "InsufficientStorage",
520: "Unknown",
}

codeDetails = map[int]string{
507: "/var is almost out of disk space",
}
)

// Status holds string representations of component states
Expand Down Expand Up @@ -258,7 +265,6 @@ func exitCode(statuses []*Status) int {

// nodeStatus looks up the status of a node
func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*Status, error) {

controlPlane := n.ControlPlane
name := driver.MachineName(cc, n)

Expand Down Expand Up @@ -315,6 +321,17 @@ func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*St
return st, err
}

// Check storage
p, err := machine.DiskUsed(cr, "/var")
if err != nil {
glog.Errorf("failed to get storage capacity of /var: %v", err)
st.Host = state.Error.String()
return st, err
}
if p >= 99 {
st.Host = codeNames[InsufficientStorage]
}

stk := kverify.KubeletStatus(cr)
glog.Infof("%s kubelet status = %s", name, stk)
st.Kubelet = stk.String()
Expand Down Expand Up @@ -423,12 +440,15 @@ func readEventLog(name string) ([]cloudevents.Event, time.Time, error) {

// clusterState converts Status structs into a ClusterState struct
func clusterState(sts []*Status) ClusterState {
sc := statusCode(sts[0].Host)
cs := ClusterState{
BinaryVersion: version.GetVersion(),

BaseState: BaseState{
Name: ClusterFlagValue(),
StatusCode: statusCode(sts[0].APIServer),
Name: ClusterFlagValue(),
StatusCode: sc,
StatusName: sts[0].Host,
StatusDetail: codeDetails[sc],
},

Components: map[string]BaseState{
Expand Down Expand Up @@ -499,6 +519,26 @@ func clusterState(sts []*Status) ClusterState {
finalStep = data
glog.Infof("transient code %d (%q) for step: %+v", transientCode, codeNames[transientCode], data)
}
if ev.Type() == "io.k8s.sigs.minikube.error" {
var data map[string]string
err := ev.DataAs(&data)
if err != nil {
glog.Errorf("unable to parse data: %v\nraw data: %s", err, ev.Data())
continue
}
exitCode, err := strconv.Atoi(data["exitcode"])
if err != nil {
glog.Errorf("unable to convert exit code to int: %v", err)
continue
}
transientCode = exitCode
for _, n := range cs.Nodes {
n.StatusCode = transientCode
n.StatusName = codeNames[n.StatusCode]
}

glog.Infof("transient code %d (%q) for step: %+v", transientCode, codeNames[transientCode], data)
}
}

if finalStep != nil {
Expand All @@ -514,6 +554,7 @@ func clusterState(sts []*Status) ClusterState {
}

cs.StatusName = codeNames[cs.StatusCode]
cs.StatusDetail = codeDetails[cs.StatusCode]
return cs
}

Expand Down
2 changes: 2 additions & 0 deletions pkg/minikube/constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ const (
MinikubeActivePodmanEnv = "MINIKUBE_ACTIVE_PODMAN"
// MinikubeForceSystemdEnv is used to force systemd as cgroup manager for the container runtime
MinikubeForceSystemdEnv = "MINIKUBE_FORCE_SYSTEMD"
// TestDiskUsedEnv is used in integration tests for insufficient storage with 'minikube status'
TestDiskUsedEnv = "MINIKUBE_TEST_STORAGE_CAPACITY"
)

var (
Expand Down
21 changes: 11 additions & 10 deletions pkg/minikube/exit/exit.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,17 @@ import (

// Exit codes based on sysexits(3)
const (
Failure = 1 // Failure represents a general failure code
Interrupted = 2 // Ctrl-C (SIGINT)
BadUsage = 64 // Usage represents an incorrect command line
Data = 65 // Data represents incorrect data supplied by the user
NoInput = 66 // NoInput represents that the input file did not exist or was not readable
Unavailable = 69 // Unavailable represents when a service was unavailable
Software = 70 // Software represents an internal software error.
IO = 74 // IO represents an I/O error
Config = 78 // Config represents an unconfigured or misconfigured state
Permissions = 77 // Permissions represents a permissions error
Failure = 1 // Failure represents a general failure code
Interrupted = 2 // Ctrl-C (SIGINT)
BadUsage = 64 // Usage represents an incorrect command line
Data = 65 // Data represents incorrect data supplied by the user
NoInput = 66 // NoInput represents that the input file did not exist or was not readable
Unavailable = 69 // Unavailable represents when a service was unavailable
Software = 70 // Software represents an internal software error.
IO = 74 // IO represents an I/O error
Permissions = 77 // Permissions represents a permissions error
Config = 78 // Config represents an unconfigured or misconfigured state
InsufficientStorage = 507 // InsufficientStorage represents insufficient storage in the VM/container
)

// UsageT outputs a templated usage error and exits with error code 64
Expand Down
30 changes: 22 additions & 8 deletions pkg/minikube/machine/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"encoding/json"
"fmt"
"net"
"os"
"os/exec"
"path"
"path/filepath"
Expand Down Expand Up @@ -212,24 +213,37 @@ func postStartValidations(h *host.Host, drvName string) {
if !driver.IsKIC(drvName) {
return
}
// make sure /var isn't full, otherwise warn
output, err := h.RunSSHCommand("df -h /var | awk 'NR==2{print $5}'")
r, err := CommandRunner(h)
if err != nil {
glog.Warningf("error running df -h /var: %v", err)
glog.Warningf("error getting command runner: %v", err)
}
output = strings.TrimSpace(output)
output = strings.Trim(output, "%")
percentageFull, err := strconv.Atoi(output)
// make sure /var isn't full, otherwise warn
percentageFull, err := DiskUsed(r, "/var")
if err != nil {
glog.Warningf("error getting percentage of /var that is free: %v", err)
}
if percentageFull >= 99 {
exit.WithError("", fmt.Errorf("docker daemon out of memory. No space left on device"))
exit.WithCodeT(exit.InsufficientStorage, "docker daemon out of memory. No space left on device")
}

if percentageFull > 80 {
out.WarningT("The docker daemon is almost out of memory, run 'docker system prune' to free up space")
out.ErrT(out.Tip, "The docker daemon is almost out of memory, run 'docker system prune' to free up space")
}
}

// DiskUsed returns the capacity of dir in the VM/container as a percentage
func DiskUsed(cr command.Runner, dir string) (int, error) {
if s := os.Getenv(constants.TestDiskUsedEnv); s != "" {
return strconv.Atoi(s)
}
output, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf("df -h %s | awk 'NR==2{print $5}'", dir)))
if err != nil {
glog.Warningf("error running df -h /var: %v\n%v", err, output.Output())
return 0, err
}
percentage := strings.TrimSpace(output.Stdout.String())
percentage = strings.Trim(percentage, "%")
return strconv.Atoi(percentage)
}

// postStart are functions shared between startHost and fixHost
Expand Down
2 changes: 1 addition & 1 deletion pkg/minikube/out/register/cloud_events.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func printAndRecordCloudEvent(log Log, data map[string]string) {
fmt.Fprintln(outputFile, string(bs))

if eventFile != nil {
go storeEvent(bs)
storeEvent(bs)
}
}

Expand Down
98 changes: 98 additions & 0 deletions test/integration/status_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"path"
"testing"

"k8s.io/minikube/cmd/minikube/cmd"
"k8s.io/minikube/pkg/minikube/constants"
"k8s.io/minikube/pkg/minikube/localpath"
)

func TestInsufficientStorage(t *testing.T) {
if !KicDriver() {
t.Skip("only runs with docker driver")
}
profile := UniqueProfileName("insufficient-storage")
ctx, cancel := context.WithTimeout(context.Background(), Minutes(5))
defer Cleanup(t, profile, cancel)

startArgs := []string{"start", "-p", profile, "--output=json", "--wait=true"}
startArgs = append(startArgs, StartArgs()...)
c := exec.CommandContext(ctx, Target(), startArgs...)
// artificially set /var to 100% capacity
c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestDiskUsedEnv))

rr, err := Run(t, c)
if err == nil {
t.Fatalf("expected command to fail, but it succeeded: %v\n%v", rr.Command(), err)
}

// make sure 'minikube status' has correct output
stdout := runStatusCmd(ctx, t, profile)
verifyClusterState(t, stdout)

// try deleting events.json and make sure this still works
eventsFile := path.Join(localpath.MiniPath(), "profiles", profile, "events.json")
if err := os.Remove(eventsFile); err != nil {
t.Fatalf("removing %s", eventsFile)
}
stdout = runStatusCmd(ctx, t, profile)
verifyClusterState(t, stdout)
}

// runStatusCmd runs the status command and returns stdout
func runStatusCmd(ctx context.Context, t *testing.T, profile string) []byte {
// make sure minikube status shows insufficient storage
c := exec.CommandContext(ctx, Target(), "status", "-p", profile, "--output=json", "--layout=cluster")
// artificially set /var to 100% capacity
c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestDiskUsedEnv))
rr, err := Run(t, c)
// status exits non-0 if status isn't Running
if err == nil {
t.Fatalf("expected command to fail, but it succeeded: %v\n%v", rr.Command(), err)
}
return rr.Stdout.Bytes()
}

func verifyClusterState(t *testing.T, contents []byte) {
var cs cmd.ClusterState
if err := json.Unmarshal(contents, &cs); err != nil {
t.Fatalf("unmarshalling: %v", err)
}
// verify the status looks as we expect
if cs.StatusCode != cmd.InsufficientStorage {
t.Fatalf("incorrect status code: %v", cs.StatusCode)
}
if cs.StatusName != "InsufficientStorage" {
t.Fatalf("incorrect status name: %v", cs.StatusName)
}
for _, n := range cs.Nodes {
if n.StatusCode != cmd.InsufficientStorage {
t.Fatalf("incorrect node status code: %v", cs.StatusCode)
}
}
}

0 comments on commit 7c84e2e

Please sign in to comment.