Merge pull request #3673 from tstromberg/logs-apiserver3

Include pod output in 'logs' command & display detected problems during start

cmd/minikube/cmd/logs.go

@@ -17,17 +17,28 @@ limitations under the License.
 package cmd
 
 import (
-	"os"
-
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/logs"
 	"k8s.io/minikube/pkg/minikube/machine"
 )
 
+const (
+	// number of problems per log to output
+	numberOfProblems = 5
+)
+
 var (
-	follow bool
+	// followLogs triggers tail -f mode
+	followLogs bool
+	// numberOfLines is how many lines to output, set via -n
+	numberOfLines int
+	// showProblems only shows lines that match known issues
+	showProblems bool
 )
 
 // logsCmd represents the logs command
@@ -36,24 +47,56 @@ var logsCmd = &cobra.Command{
 	Short: "Gets the logs of the running instance, used for debugging minikube, not user code",
 	Long:  `Gets the logs of the running instance, used for debugging minikube, not user code.`,
 	Run: func(cmd *cobra.Command, args []string) {
+		cfg, err := config.Load()
+		if err != nil {
+			exit.WithError("Error getting config", err)
+		}
+
 		api, err := machine.NewAPIClient()
 		if err != nil {
 			exit.WithError("Error getting client", err)
 		}
 		defer api.Close()
-		clusterBootstrapper, err := GetClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
+
+		h, err := api.Load(config.GetMachineName())
+		if err != nil {
+			exit.WithError("api load", err)
+		}
+		runner, err := machine.CommandRunner(h)
+		if err != nil {
+			exit.WithError("command runner", err)
+		}
+		bs, err := GetClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
 		if err != nil {
 			exit.WithError("Error getting cluster bootstrapper", err)
 		}
 
-		err = clusterBootstrapper.GetClusterLogsTo(follow, os.Stdout)
+		cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: runner})
+		if err != nil {
+			exit.WithError("Unable to get runtime", err)
+		}
+		if followLogs {
+			err := logs.Follow(cr, bs, runner)
+			if err != nil {
+				exit.WithError("Follow", err)
+			}
+			return
+		}
+		if showProblems {
+			problems := logs.FindProblems(cr, bs, runner)
+			logs.OutputProblems(problems, numberOfProblems)
+			return
+		}
+		err = logs.Output(cr, bs, runner, numberOfLines)
 		if err != nil {
 			exit.WithError("Error getting machine logs", err)
 		}
 	},
 }
 
 func init() {
-	logsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.")
+	logsCmd.Flags().BoolVarP(&followLogs, "follow", "f", false, "Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.")
+	logsCmd.Flags().BoolVar(&showProblems, "problems", false, "Show only log entries which point to known problems")
+	logsCmd.Flags().IntVarP(&numberOfLines, "length", "n", 50, "Number of lines back to go within the log")
 	RootCmd.AddCommand(logsCmd)
 }

cmd/minikube/cmd/start.go

@@ -46,6 +46,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/logs"
 	"k8s.io/minikube/pkg/minikube/machine"
 	pkgutil "k8s.io/minikube/pkg/util"
 	"k8s.io/minikube/pkg/util/kubeconfig"
@@ -187,15 +188,19 @@ func runStart(cmd *cobra.Command, args []string) {
 	if err := saveConfig(config); err != nil {
 		exit.WithError("Failed to save config", err)
 	}
+	runner, err := machine.CommandRunner(host)
+	if err != nil {
+		exit.WithError("Failed to get command runner", err)
+	}
 
-	configureRuntimes(host)
+	cr := configureRuntimes(host, runner)
 	bs := prepareHostEnvironment(m, config.KubernetesConfig)
 	waitCacheImages(&cacheGroup)
 
 	// The kube config must be update must come before bootstrapping, otherwise health checks may use a stale IP
 	kubeconfig := updateKubeConfig(host, &config)
-	bootstrapCluster(bs, config.KubernetesConfig, preexisting)
-	validateCluster(bs, ip)
+	bootstrapCluster(bs, cr, runner, config.KubernetesConfig, preexisting)
+	validateCluster(bs, cr, runner, ip)
 	configureMounts()
 	if err = LoadCachedImagesInConfigFile(); err != nil {
 		console.Failure("Unable to load cached images from config file.")
@@ -446,12 +451,7 @@ func updateKubeConfig(h *host.Host, c *cfg.Config) *kubeconfig.KubeConfigSetup {
 }
 
 // configureRuntimes does what needs to happen to get a runtime going.
-func configureRuntimes(h *host.Host) {
-	runner, err := machine.CommandRunner(h)
-	if err != nil {
-		exit.WithError("Failed to get command runner", err)
-	}
-
+func configureRuntimes(h *host.Host, runner bootstrapper.CommandRunner) cruntime.Manager {
 	config := cruntime.Config{Type: viper.GetString(containerRuntime), Runner: runner}
 	cr, err := cruntime.New(config)
 	if err != nil {
@@ -469,7 +469,7 @@ func configureRuntimes(h *host.Host) {
 	if err != nil {
 		exit.WithError("Failed to enable container runtime", err)
 	}
-
+	return cr
 }
 
 // waitCacheImages blocks until the image cache jobs complete
@@ -484,7 +484,7 @@ func waitCacheImages(g *errgroup.Group) {
 }
 
 // bootstrapCluster starts Kubernetes using the chosen bootstrapper
-func bootstrapCluster(bs bootstrapper.Bootstrapper, kc cfg.KubernetesConfig, preexisting bool) {
+func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner bootstrapper.CommandRunner, kc cfg.KubernetesConfig, preexisting bool) {
 	console.OutStyle("pulling", "Pulling images used by Kubernetes %s ...", kc.KubernetesVersion)
 	if err := bs.PullImages(kc); err != nil {
 		console.OutStyle("failure", "Unable to pull images, which may be OK: %v", err)
@@ -495,19 +495,19 @@ func bootstrapCluster(bs bootstrapper.Bootstrapper, kc cfg.KubernetesConfig, pre
 	if preexisting {
 		console.OutStyle("restarting", "Relaunching Kubernetes %s using %s ... ", kc.KubernetesVersion, bsName)
 		if err := bs.RestartCluster(kc); err != nil {
-			exit.WithError("Error restarting cluster", err)
+			exit.WithProblems("Error restarting cluster", err, logs.FindProblems(r, bs, runner))
 		}
 		return
 	}
 
 	console.OutStyle("launch", "Launching Kubernetes %s using %s ... ", kc.KubernetesVersion, bsName)
 	if err := bs.StartCluster(kc); err != nil {
-		exit.WithError("Error starting cluster", err)
+		exit.WithProblems("Error starting cluster", err, logs.FindProblems(r, bs, runner))
 	}
 }
 
 // validateCluster validates that the cluster is well-configured and healthy
-func validateCluster(bs bootstrapper.Bootstrapper, ip string) {
+func validateCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner bootstrapper.CommandRunner, ip string) {
 	console.OutStyle("verifying-noline", "Verifying component health ...")
 	kStat := func() (err error) {
 		st, err := bs.GetKubeletStatus()
@@ -519,7 +519,7 @@ func validateCluster(bs bootstrapper.Bootstrapper, ip string) {
 	}
 	err := pkgutil.RetryAfter(20, kStat, 3*time.Second)
 	if err != nil {
-		exit.WithError("kubelet checks failed", err)
+		exit.WithProblems("kubelet checks failed", err, logs.FindProblems(r, bs, runner))
 	}
 	aStat := func() (err error) {
 		st, err := bs.GetApiServerStatus(net.ParseIP(ip))
@@ -532,7 +532,7 @@ func validateCluster(bs bootstrapper.Bootstrapper, ip string) {
 
 	err = pkgutil.RetryAfter(30, aStat, 10*time.Second)
 	if err != nil {
-		exit.WithError("apiserver checks failed", err)
+		exit.WithProblems("apiserver checks failed", err, logs.FindProblems(r, bs, runner))
 	}
 	console.OutLn("")
 }

pkg/minikube/bootstrapper/bootstrapper.go

@@ -17,13 +17,20 @@ limitations under the License.
 package bootstrapper
 
 import (
-	"io"
 	"net"
 
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
+// LogOptions are options to be passed to LogCommands
+type LogOptions struct {
+	// Lines is the number of recent log lines to include, as in tail -n.
+	Lines int
+	// Follow is whether or not to actively follow the logs, as in tail -f.
+	Follow bool
+}
+
 // Bootstrapper contains all the methods needed to bootstrap a kubernetes cluster
 type Bootstrapper interface {
 	// PullImages pulls images necessary for a cluster. Success should not be required.
@@ -32,7 +39,8 @@ type Bootstrapper interface {
 	UpdateCluster(config.KubernetesConfig) error
 	RestartCluster(config.KubernetesConfig) error
 	DeleteCluster(config.KubernetesConfig) error
-	GetClusterLogsTo(follow bool, out io.Writer) error
+	// LogCommands returns a map of log type to a command which will display that log.
+	LogCommands(LogOptions) map[string]string
 	SetupCerts(cfg config.KubernetesConfig) error
 	GetKubeletStatus() (string, error)
 	GetApiServerStatus(net.IP) (string, error)

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -21,7 +21,6 @@ import (
 	"crypto"
 	"crypto/tls"
 	"fmt"
-	"io"
 	"net"
 	"net/http"
 	"os"
@@ -121,28 +120,17 @@ func (k *KubeadmBootstrapper) GetApiServerStatus(ip net.IP) (string, error) {
 	return state.Running.String(), nil
 }
 
-// TODO(r2d4): Should this aggregate all the logs from the control plane?
-// Maybe subcommands for each component? minikube logs apiserver?
-func (k *KubeadmBootstrapper) GetClusterLogsTo(follow bool, out io.Writer) error {
-	var flags []string
-	if follow {
-		flags = append(flags, "-f")
+// LogCommands returns a map of log type to a command which will display that log.
+func (k *KubeadmBootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string {
+	var kcmd strings.Builder
+	kcmd.WriteString("journalctl -u kubelet")
+	if o.Lines > 0 {
+		kcmd.WriteString(fmt.Sprintf(" -n %d", o.Lines))
 	}
-	logsCommand := fmt.Sprintf("sudo journalctl %s -u kubelet", strings.Join(flags, " "))
-
-	if follow {
-		if err := k.c.CombinedOutputTo(logsCommand, out); err != nil {
-			return errors.Wrap(err, "getting cluster logs")
-		}
-	} else {
-
-		logs, err := k.c.CombinedOutput(logsCommand)
-		if err != nil {
-			return errors.Wrap(err, "getting cluster logs")
-		}
-		fmt.Fprint(out, logs)
+	if o.Follow {
+		kcmd.WriteString(" -f")
 	}
-	return nil
+	return map[string]string{"kubelet": kcmd.String()}
 }
 
 func (k *KubeadmBootstrapper) StartCluster(k8s config.KubernetesConfig) error {

pkg/minikube/console/style.go

@@ -40,20 +40,21 @@ var styles = map[string]style{
 	"fatal":        {Prefix: "💣  "},
 	"notice":       {Prefix: "📌  "},
 	"ready":        {Prefix: "🏄  "},
-	"running":      {Prefix: "🏃  "},
-	"provisioning": {Prefix: "🌱  "},
 	"restarting":   {Prefix: "🔄  "},
 	"stopping":     {Prefix: "✋  "},
 	"stopped":      {Prefix: "🛑  "},
 	"warning":      {Prefix: "⚠️  "},
 	"waiting":      {Prefix: "⌛  "},
 	"usage":        {Prefix: "💡  "},
 	"launch":       {Prefix: "🚀  "},
-	"sad":          {Prefix: "😿  "},
 	"thumbs-up":    {Prefix: "👍  "},
 	"option":       {Prefix: "    ▪ "}, // Indented bullet
-	"url":          {Prefix: "👉  "},
+	"log-entry":    {Prefix: "    "},   // Indent
 	"crushed":      {Prefix: "💔  "},
+	"running":      {Prefix: "🏃  "},
+	"provisioning": {Prefix: "🌱  "},
+	"sad":          {Prefix: "😿  "},
+	"url":          {Prefix: "👉  "},
 
 	// Specialized purpose styles
 	"iso-download":      {Prefix: "💿  "},

pkg/minikube/cruntime/containerd.go

@@ -107,3 +107,8 @@ func (r *Containerd) KillContainers(ids []string) error {
 func (r *Containerd) StopContainers(ids []string) error {
 	return stopCRIContainers(r.Runner, ids)
 }
+
+// ContainerLogCmd returns the command to retrieve the log for a container based on ID
+func (r *Containerd) ContainerLogCmd(id string, len int, follow bool) string {
+	return criContainerLogCmd(id, len, follow)
+}

pkg/minikube/cruntime/cri.go

@@ -60,3 +60,18 @@ image-endpoint: unix://{{.Socket}}
 	}
 	return cr.Run(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(cPath), b.String(), cPath))
 }
+
+// criContainerLogCmd returns the command to retrieve the log for a container based on ID
+func criContainerLogCmd(id string, len int, follow bool) string {
+	var cmd strings.Builder
+	cmd.WriteString("crictl logs ")
+	if len > 0 {
+		cmd.WriteString(fmt.Sprintf("--tail %d ", len))
+	}
+	if follow {
+		cmd.WriteString("--follow ")
+	}
+
+	cmd.WriteString(id)
+	return cmd.String()
+}

pkg/minikube/cruntime/crio.go

@@ -106,3 +106,8 @@ func (r *CRIO) KillContainers(ids []string) error {
 func (r *CRIO) StopContainers(ids []string) error {
 	return stopCRIContainers(r.Runner, ids)
 }
+
+// ContainerLogCmd returns the command to retrieve the log for a container based on ID
+func (r *CRIO) ContainerLogCmd(id string, len int, follow bool) string {
+	return criContainerLogCmd(id, len, follow)
+}

pkg/minikube/cruntime/cruntime.go

@@ -61,6 +61,8 @@ type Manager interface {
 	KillContainers([]string) error
 	// StopContainers stops containers based on ID
 	StopContainers([]string) error
+	// ContainerLogCmd returns the command to retrieve the log for a container based on ID
+	ContainerLogCmd(string, int, bool) string
 }
 
 // Config is runtime configuration

pkg/minikube/cruntime/docker.go

@@ -101,3 +101,18 @@ func (r *Docker) KillContainers(ids []string) error {
 func (r *Docker) StopContainers(ids []string) error {
 	return r.Runner.Run(fmt.Sprintf("docker stop %s", strings.Join(ids, " ")))
 }
+
+// ContainerLogCmd returns the command to retrieve the log for a container based on ID
+func (r *Docker) ContainerLogCmd(id string, len int, follow bool) string {
+	var cmd strings.Builder
+	cmd.WriteString("docker logs ")
+	if len > 0 {
+		cmd.WriteString(fmt.Sprintf("--tail %d ", len))
+	}
+	if follow {
+		cmd.WriteString("--follow ")
+	}
+
+	cmd.WriteString(id)
+	return cmd.String()
+}