From a7c49e47c1670e4591d5fc37c08a1ae59c7f1a5e Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Thu, 21 May 2020 12:46:13 -0700 Subject: [PATCH 1/6] allow minikube status to display for one node --- cmd/minikube/cmd/node_start.go | 2 +- cmd/minikube/cmd/node_stop.go | 2 +- cmd/minikube/cmd/ssh.go | 2 +- cmd/minikube/cmd/status.go | 32 ++++++++++++++++++++++++-------- pkg/minikube/node/node.go | 10 ++++++++-- test/integration/helpers.go | 7 ++++--- 6 files changed, 39 insertions(+), 16 deletions(-) diff --git a/cmd/minikube/cmd/node_start.go b/cmd/minikube/cmd/node_start.go index 4710599c93a4..daac52a4a46e 100644 --- a/cmd/minikube/cmd/node_start.go +++ b/cmd/minikube/cmd/node_start.go @@ -40,7 +40,7 @@ var nodeStartCmd = &cobra.Command{ api, cc := mustload.Partial(ClusterFlagValue()) name := args[0] - n, _, err := node.Retrieve(cc, name) + n, _, err := node.Retrieve(*cc, name) if err != nil { exit.WithError("retrieving node", err) } diff --git a/cmd/minikube/cmd/node_stop.go b/cmd/minikube/cmd/node_stop.go index 771f293bea07..9108bc65b853 100644 --- a/cmd/minikube/cmd/node_stop.go +++ b/cmd/minikube/cmd/node_stop.go @@ -38,7 +38,7 @@ var nodeStopCmd = &cobra.Command{ name := args[0] api, cc := mustload.Partial(ClusterFlagValue()) - n, _, err := node.Retrieve(cc, name) + n, _, err := node.Retrieve(*cc, name) if err != nil { exit.WithError("retrieving node", err) } diff --git a/cmd/minikube/cmd/ssh.go b/cmd/minikube/cmd/ssh.go index 703119740c50..d365cae360f7 100644 --- a/cmd/minikube/cmd/ssh.go +++ b/cmd/minikube/cmd/ssh.go @@ -52,7 +52,7 @@ var sshCmd = &cobra.Command{ if nodeName == "" { n = co.CP.Node } else { - n, _, err = node.Retrieve(co.Config, nodeName) + n, _, err = node.Retrieve(*co.Config, nodeName) if err != nil { exit.WithCodeT(exit.Unavailable, "Node {{.nodeName}} does not exist.", out.V{"nodeName": nodeName}) } diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index 8e6f48e49120..fde5f1c5758a 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -37,6 +37,7 @@ import ( "k8s.io/minikube/pkg/minikube/kubeconfig" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/mustload" + "k8s.io/minikube/pkg/minikube/node" ) var statusFormat string @@ -105,19 +106,33 @@ var statusCmd = &cobra.Command{ api, cc := mustload.Partial(cname) var statuses []*Status - for _, n := range cc.Nodes { - glog.Infof("checking status of %s ...", n.Name) - machineName := driver.MachineName(*cc, n) - st, err := status(api, *cc, n) - glog.Infof("%s status: %+v", machineName, st) + if nodeName != "" || statusFormat != defaultStatusFormat { + n, _, err := node.Retrieve(*cc, nodeName) if err != nil { - glog.Errorf("status error: %v", err) + exit.WithError("retrieving node", err) } - if st.Host == Nonexistent { - glog.Errorf("The %q host does not exist!", machineName) + + st, err := status(api, *cc, *n) + if err != nil { + glog.Errorf("status error: %v", err) } statuses = append(statuses, st) + } else { + for _, n := range cc.Nodes { + glog.Infof("checking status of %s ...", n.Name) + machineName := driver.MachineName(*cc, n) + st, err := status(api, *cc, n) + glog.Infof("%s status: %+v", machineName, st) + + if err != nil { + glog.Errorf("status error: %v", err) + } + if st.Host == Nonexistent { + glog.Errorf("The %q host does not exist!", machineName) + } + statuses = append(statuses, st) + } } switch strings.ToLower(output) { @@ -253,6 +268,7 @@ func init() { For the list accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#Status`) statusCmd.Flags().StringVarP(&output, "output", "o", "text", `minikube status --output OUTPUT. json, text`) + statusCmd.Flags().StringVarP(&nodeName, "node", "n", "", "The node to check status for. Defaults to control plane. Leave blank with default format for status on all nodes.") } func statusText(st *Status, w io.Writer) error { diff --git a/pkg/minikube/node/node.go b/pkg/minikube/node/node.go index 74cb8f69d7ec..ada4de12964f 100644 --- a/pkg/minikube/node/node.go +++ b/pkg/minikube/node/node.go @@ -59,7 +59,7 @@ func Add(cc *config.ClusterConfig, n config.Node) error { // Delete stops and deletes the given node from the given cluster func Delete(cc config.ClusterConfig, name string) (*config.Node, error) { - n, index, err := Retrieve(&cc, name) + n, index, err := Retrieve(cc, name) if err != nil { return n, errors.Wrap(err, "retrieve") } @@ -79,11 +79,17 @@ func Delete(cc config.ClusterConfig, name string) (*config.Node, error) { } // Retrieve finds the node by name in the given cluster -func Retrieve(cc *config.ClusterConfig, name string) (*config.Node, int, error) { +func Retrieve(cc config.ClusterConfig, name string) (*config.Node, int, error) { + for i, n := range cc.Nodes { if n.Name == name { return &n, i, nil } + + // Accept full machine name as well as just node name + if driver.MachineName(cc, n) == name { + return &n, i, nil + } } return nil, -1, errors.New("Could not find node " + name) diff --git a/test/integration/helpers.go b/test/integration/helpers.go index 60da0e916655..b997a035f69e 100644 --- a/test/integration/helpers.go +++ b/test/integration/helpers.go @@ -191,13 +191,14 @@ func CleanupWithLogs(t *testing.T, profile string, cancel context.CancelFunc) { } // PostMortemLogs shows logs for debugging a failed cluster -func PostMortemLogs(t *testing.T, profile string) { +func PostMortemLogs(t *testing.T, profile string, node ...string) { if !t.Failed() { return } if !*postMortemLogs { - t.Logf("post-mortem logs disabled, oh-well!") + t.Logf("post-mortem logs disabled, oh well!") + return } t.Logf("-----------------------post-mortem--------------------------------") @@ -355,7 +356,7 @@ func PodWait(ctx context.Context, t *testing.T, profile string, ns string, selec } // Status returns a minikube component status as a string -func Status(ctx context.Context, t *testing.T, path string, profile string, key string) string { +func Status(ctx context.Context, t *testing.T, path string, profile string, key string, node ...string) string { t.Helper() // Reminder of useful keys: "Host", "Kubelet", "APIServer" rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile)) From 95755c07fdb8e17096934ce147cb1ab7245cf267 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Thu, 21 May 2020 12:54:54 -0700 Subject: [PATCH 2/6] generate docs --- site/content/en/docs/commands/status.md | 1 + 1 file changed, 1 insertion(+) diff --git a/site/content/en/docs/commands/status.md b/site/content/en/docs/commands/status.md index 1d4930b95a9f..c65a79531ce0 100644 --- a/site/content/en/docs/commands/status.md +++ b/site/content/en/docs/commands/status.md @@ -26,6 +26,7 @@ minikube status [flags] -f, --format string Go template format string for the status output. The format for Go templates can be found here: https://golang.org/pkg/text/template/ For the list accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#Status (default "{{.Name}}\ntype: Control Plane\nhost: {{.Host}}\nkubelet: {{.Kubelet}}\napiserver: {{.APIServer}}\nkubeconfig: {{.Kubeconfig}}\n\n") -h, --help help for status + -n, --node string The node to check status for. Defaults to control plane. Leave blank with default format for status on all nodes. -o, --output string minikube status --output OUTPUT. json, text (default "text") ``` From 9fe9513c345258a90093c21e4bc80cb18ff3d6a6 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Thu, 21 May 2020 14:08:34 -0700 Subject: [PATCH 3/6] only use retrieval code for multinode clusters --- .travis.yml | 2 -- cmd/minikube/cmd/status.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9b54778ac684..61fb1ef1f731 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,8 +33,6 @@ matrix: script: make after_success: - bash <(curl -s https://codecov.io/bash) -travisBuddy: - regex: (FAIL:|\.go:\d+:|^panic:|failed$) notifications: webhooks: urls: diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index fde5f1c5758a..945e11d52c43 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -107,7 +107,7 @@ var statusCmd = &cobra.Command{ var statuses []*Status - if nodeName != "" || statusFormat != defaultStatusFormat { + if nodeName != "" || statusFormat != defaultStatusFormat && len(cc.Nodes) > 1 { n, _, err := node.Retrieve(*cc, nodeName) if err != nil { exit.WithError("retrieving node", err) From 70c228c2c3a212cc8cb272134fe75630e282c4ef Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Thu, 21 May 2020 14:29:33 -0700 Subject: [PATCH 4/6] adding postmortem logs for worker nodes in integration tests --- test/integration/helpers.go | 106 +++++++++++++++++------------ test/integration/main.go | 6 ++ test/integration/multinode_test.go | 16 ++--- 3 files changed, 72 insertions(+), 56 deletions(-) diff --git a/test/integration/helpers.go b/test/integration/helpers.go index b997a035f69e..cc3036a7e938 100644 --- a/test/integration/helpers.go +++ b/test/integration/helpers.go @@ -191,7 +191,7 @@ func CleanupWithLogs(t *testing.T, profile string, cancel context.CancelFunc) { } // PostMortemLogs shows logs for debugging a failed cluster -func PostMortemLogs(t *testing.T, profile string, node ...string) { +func PostMortemLogs(t *testing.T, profile string, multinode ...bool) { if !t.Failed() { return } @@ -201,57 +201,73 @@ func PostMortemLogs(t *testing.T, profile string, node ...string) { return } + m := false + if len(multinode) > 0 { + m = multinode[0] + } + + nodes := []string{profile} + if m { + nodes = append(nodes, SecondNodeName, ThirdNodeName) + } + t.Logf("-----------------------post-mortem--------------------------------") - if DockerDriver() { - t.Logf("======> post-mortem[%s]: docker inspect <======", t.Name()) - rr, err := Run(t, exec.Command("docker", "inspect", profile)) - if err != nil { - t.Logf("failed to get docker inspect: %v", err) - } else { - t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) + for _, n := range nodes { + machine := profile + if n != profile { + machine = fmt.Sprintf("%s-%s", profile, n) + } + if DockerDriver() { + t.Logf("======> post-mortem[%s]: docker inspect <======", t.Name()) + rr, err := Run(t, exec.Command("docker", "inspect", machine)) + if err != nil { + t.Logf("failed to get docker inspect: %v", err) + } else { + t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) + } } - } - st := Status(context.Background(), t, Target(), profile, "Host") - if st != state.Running.String() { - t.Logf("%q host is not running, skipping log retrieval (state=%q)", profile, st) - return - } - t.Logf("<<< %s FAILED: start of post-mortem logs <<<", t.Name()) - t.Logf("======> post-mortem[%s]: minikube logs <======", t.Name()) + st := Status(context.Background(), t, Target(), profile, "Host", n) + if st != state.Running.String() { + t.Logf("%q host is not running, skipping log retrieval (state=%q)", profile, st) + return + } + t.Logf("<<< %s FAILED: start of post-mortem logs <<<", t.Name()) + t.Logf("======> post-mortem[%s]: minikube logs <======", t.Name()) - rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "-n", "25")) - if err != nil { - t.Logf("failed logs error: %v", err) - return - } - t.Logf("%s logs: %s", t.Name(), rr.Output()) + rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "-n", "25")) + if err != nil { + t.Logf("failed logs error: %v", err) + return + } + t.Logf("%s logs: %s", t.Name(), rr.Output()) - st = Status(context.Background(), t, Target(), profile, "APIServer") - if st != state.Running.String() { - t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st) - return - } + st = Status(context.Background(), t, Target(), profile, "APIServer", n) + if st != state.Running.String() { + t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st) + return + } - // Get non-running pods. NOTE: This does not yet contain pods which are "running", but not "ready" - rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-o=jsonpath={.items[*].metadata.name}", "-A", "--field-selector=status.phase!=Running")) - if rerr != nil { - t.Logf("%s: %v", rr.Command(), rerr) - return - } - notRunning := strings.Split(rr.Stdout.String(), " ") - t.Logf("non-running pods: %s", strings.Join(notRunning, " ")) + // Get non-running pods. NOTE: This does not yet contain pods which are "running", but not "ready" + rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-o=jsonpath={.items[*].metadata.name}", "-A", "--field-selector=status.phase!=Running")) + if rerr != nil { + t.Logf("%s: %v", rr.Command(), rerr) + return + } + notRunning := strings.Split(rr.Stdout.String(), " ") + t.Logf("non-running pods: %s", strings.Join(notRunning, " ")) - t.Logf("======> post-mortem[%s]: describe non-running pods <======", t.Name()) + t.Logf("======> post-mortem[%s]: describe non-running pods <======", t.Name()) - args := append([]string{"--context", profile, "describe", "pod"}, notRunning...) - rr, rerr = Run(t, exec.Command("kubectl", args...)) - if rerr != nil { - t.Logf("%s: %v", rr.Command(), rerr) - return + args := append([]string{"--context", profile, "describe", "pod"}, notRunning...) + rr, rerr = Run(t, exec.Command("kubectl", args...)) + if rerr != nil { + t.Logf("%s: %v", rr.Command(), rerr) + return + } + t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) } - t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) t.Logf("<<< %s FAILED: end of post-mortem logs <<<", t.Name()) t.Logf("---------------------/post-mortem---------------------------------") @@ -356,10 +372,10 @@ func PodWait(ctx context.Context, t *testing.T, profile string, ns string, selec } // Status returns a minikube component status as a string -func Status(ctx context.Context, t *testing.T, path string, profile string, key string, node ...string) string { +func Status(ctx context.Context, t *testing.T, path string, profile string, key string, node string) string { t.Helper() // Reminder of useful keys: "Host", "Kubelet", "APIServer" - rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile)) + rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile, "-n", node)) if err != nil { t.Logf("status error: %v (may be ok)", err) } @@ -369,7 +385,7 @@ func Status(ctx context.Context, t *testing.T, path string, profile string, key // showPodLogs logs debug info for pods func showPodLogs(ctx context.Context, t *testing.T, profile string, ns string, names []string) { t.Helper() - st := Status(context.Background(), t, Target(), profile, "APIServer") + st := Status(context.Background(), t, Target(), profile, "APIServer", profile) if st != state.Running.String() { t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st) return diff --git a/test/integration/main.go b/test/integration/main.go index fb706aebddb6..7a2865fd0878 100644 --- a/test/integration/main.go +++ b/test/integration/main.go @@ -40,6 +40,12 @@ var timeOutMultiplier = flag.Float64("timeout-multiplier", 1, "multiply the time var binaryPath = flag.String("binary", "../../out/minikube", "path to minikube binary") var testdataDir = flag.String("testdata-dir", "testdata", "the directory relative to test/integration where the testdata lives") +// Node names are consistent, let's store these for easy access later +const ( + SecondNodeName = "m02" + ThirdNodeName = "m03" +) + // TestMain is the test main func TestMain(m *testing.M) { flag.Parse() diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index 8ddf42b5f03d..7e09c180e68e 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -50,6 +50,7 @@ func TestMultiNode(t *testing.T) { for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { + defer PostMortemLogs(t, profile) tc.validator(ctx, t, profile) }) } @@ -104,11 +105,8 @@ func validateAddNodeToMultiNode(ctx context.Context, t *testing.T, profile strin } func validateStopRunningNode(ctx context.Context, t *testing.T, profile string) { - // Names are autogenerated using the node.Name() function - name := "m03" - // Run minikube node stop on that node - rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "stop", name)) + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "stop", ThirdNodeName)) if err != nil { t.Errorf("node stop returned an error. args %q: %v", rr.Command(), err) } @@ -143,11 +141,8 @@ func validateStartNodeAfterStop(ctx context.Context, t *testing.T, profile strin // TODO (#7496): remove skip once restarts work t.Skip("Restarting nodes is broken :(") - // Grab the stopped node - name := "m03" - // Start the node back up - rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "start", name)) + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "start", ThirdNodeName)) if err != nil { t.Errorf("node start returned an error. args %q: %v", rr.Command(), err) } @@ -168,10 +163,9 @@ func validateStartNodeAfterStop(ctx context.Context, t *testing.T, profile strin } func validateDeleteNodeFromMultiNode(ctx context.Context, t *testing.T, profile string) { - name := "m03" // Start the node back up - rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "delete", name)) + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "delete", ThirdNodeName)) if err != nil { t.Errorf("node stop returned an error. args %q: %v", rr.Command(), err) } @@ -195,7 +189,7 @@ func validateDeleteNodeFromMultiNode(ctx context.Context, t *testing.T, profile if err != nil { t.Errorf("failed to run %q : %v", rr.Command(), err) } - if strings.Contains(rr.Stdout.String(), fmt.Sprintf("%s-%s", profile, name)) { + if strings.Contains(rr.Stdout.String(), fmt.Sprintf("%s-%s", profile, ThirdNodeName)) { t.Errorf("docker volume was not properly deleted: %s", rr.Stdout.String()) } } From 39f6806626bca37f547878d55ff187f4ab799792 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Thu, 21 May 2020 14:35:46 -0700 Subject: [PATCH 5/6] add info log for retrieving node by machine name --- pkg/minikube/node/node.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/minikube/node/node.go b/pkg/minikube/node/node.go index ada4de12964f..bdc814526933 100644 --- a/pkg/minikube/node/node.go +++ b/pkg/minikube/node/node.go @@ -19,6 +19,7 @@ package node import ( "fmt" + "github.com/golang/glog" "github.com/pkg/errors" "github.com/spf13/viper" @@ -88,6 +89,7 @@ func Retrieve(cc config.ClusterConfig, name string) (*config.Node, int, error) { // Accept full machine name as well as just node name if driver.MachineName(cc, n) == name { + glog.Infof("Couldn't find node name %s, but found it as a machine name, returning it anyway.", name) return &n, i, nil } } From 3fc412bcc297c3d17c96ab791f078d856be7c498 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Thu, 21 May 2020 14:44:47 -0700 Subject: [PATCH 6/6] fix status calls in tests --- test/integration/start_stop_delete_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index 2462c456f2c2..ade1fed0655c 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -171,7 +171,7 @@ func validateEnableAddonAfterStop(ctx context.Context, t *testing.T, profile str defer PostMortemLogs(t, profile) // The none driver never really stops if !NoneDriver() { - got := Status(ctx, t, Target(), profile, "Host") + got := Status(ctx, t, Target(), profile, "Host", profile) if got != state.Stopped.String() { t.Errorf("expected post-stop host status to be -%q- but got *%q*", state.Stopped, got) } @@ -193,7 +193,7 @@ func validateSecondStart(ctx context.Context, t *testing.T, profile string, tcNa t.Fatalf("failed to start minikube post-stop. args %q: %v", rr.Command(), err) } - got := Status(ctx, t, Target(), profile, "Host") + got := Status(ctx, t, Target(), profile, "Host", profile) if got != state.Running.String() { t.Errorf("expected host status after start-stop-start to be -%q- but got *%q*", state.Running, got) } @@ -323,12 +323,12 @@ func testPause(ctx context.Context, t *testing.T, profile string) { t.Fatalf("%s failed: %v", rr.Command(), err) } - got := Status(ctx, t, Target(), profile, "APIServer") + got := Status(ctx, t, Target(), profile, "APIServer", profile) if got != state.Paused.String() { t.Errorf("post-pause apiserver status = %q; want = %q", got, state.Paused) } - got = Status(ctx, t, Target(), profile, "Kubelet") + got = Status(ctx, t, Target(), profile, "Kubelet", profile) if got != state.Stopped.String() { t.Errorf("post-pause kubelet status = %q; want = %q", got, state.Stopped) } @@ -338,12 +338,12 @@ func testPause(ctx context.Context, t *testing.T, profile string) { t.Fatalf("%s failed: %v", rr.Command(), err) } - got = Status(ctx, t, Target(), profile, "APIServer") + got = Status(ctx, t, Target(), profile, "APIServer", profile) if got != state.Running.String() { t.Errorf("post-unpause apiserver status = %q; want = %q", got, state.Running) } - got = Status(ctx, t, Target(), profile, "Kubelet") + got = Status(ctx, t, Target(), profile, "Kubelet", profile) if got != state.Running.String() { t.Errorf("post-unpause kubelet status = %q; want = %q", got, state.Running) }