diff --git a/cmd/minikube/cmd/logs.go b/cmd/minikube/cmd/logs.go index 5340cd0c5086..2df75e4e6a47 100644 --- a/cmd/minikube/cmd/logs.go +++ b/cmd/minikube/cmd/logs.go @@ -17,17 +17,28 @@ limitations under the License. package cmd import ( - "os" - "github.com/spf13/cobra" "github.com/spf13/viper" cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/logs" "k8s.io/minikube/pkg/minikube/machine" ) +const ( + // number of problems per log to output + numberOfProblems = 5 +) + var ( - follow bool + // followLogs triggers tail -f mode + followLogs bool + // numberOfLines is how many lines to output, set via -n + numberOfLines int + // showProblems only shows lines that match known issues + showProblems bool ) // logsCmd represents the logs command @@ -36,17 +47,47 @@ var logsCmd = &cobra.Command{ Short: "Gets the logs of the running instance, used for debugging minikube, not user code", Long: `Gets the logs of the running instance, used for debugging minikube, not user code.`, Run: func(cmd *cobra.Command, args []string) { + cfg, err := config.Load() + if err != nil { + exit.WithError("Error getting config", err) + } + api, err := machine.NewAPIClient() if err != nil { exit.WithError("Error getting client", err) } defer api.Close() - clusterBootstrapper, err := GetClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper)) + + h, err := api.Load(config.GetMachineName()) + if err != nil { + exit.WithError("api load", err) + } + runner, err := machine.CommandRunner(h) + if err != nil { + exit.WithError("command runner", err) + } + bs, err := GetClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper)) if err != nil { exit.WithError("Error getting cluster bootstrapper", err) } - err = clusterBootstrapper.GetClusterLogsTo(follow, os.Stdout) + cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: runner}) + if err != nil { + exit.WithError("Unable to get runtime", err) + } + if followLogs { + err := logs.Follow(cr, bs, runner) + if err != nil { + exit.WithError("Follow", err) + } + return + } + if showProblems { + problems := logs.FindProblems(cr, bs, runner) + logs.OutputProblems(problems, numberOfProblems) + return + } + err = logs.Output(cr, bs, runner, numberOfLines) if err != nil { exit.WithError("Error getting machine logs", err) } @@ -54,6 +95,8 @@ var logsCmd = &cobra.Command{ } func init() { - logsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.") + logsCmd.Flags().BoolVarP(&followLogs, "follow", "f", false, "Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.") + logsCmd.Flags().BoolVar(&showProblems, "problems", false, "Show only log entries which point to known problems") + logsCmd.Flags().IntVarP(&numberOfLines, "length", "n", 50, "Number of lines back to go within the log") RootCmd.AddCommand(logsCmd) } diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 1ede9770037a..7fd875f0f94c 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -46,6 +46,7 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/logs" "k8s.io/minikube/pkg/minikube/machine" pkgutil "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/kubeconfig" @@ -187,15 +188,19 @@ func runStart(cmd *cobra.Command, args []string) { if err := saveConfig(config); err != nil { exit.WithError("Failed to save config", err) } + runner, err := machine.CommandRunner(host) + if err != nil { + exit.WithError("Failed to get command runner", err) + } - configureRuntimes(host) + cr := configureRuntimes(host, runner) bs := prepareHostEnvironment(m, config.KubernetesConfig) waitCacheImages(&cacheGroup) // The kube config must be update must come before bootstrapping, otherwise health checks may use a stale IP kubeconfig := updateKubeConfig(host, &config) - bootstrapCluster(bs, config.KubernetesConfig, preexisting) - validateCluster(bs, ip) + bootstrapCluster(bs, cr, runner, config.KubernetesConfig, preexisting) + validateCluster(bs, cr, runner, ip) configureMounts() if err = LoadCachedImagesInConfigFile(); err != nil { console.Failure("Unable to load cached images from config file.") @@ -446,12 +451,7 @@ func updateKubeConfig(h *host.Host, c *cfg.Config) *kubeconfig.KubeConfigSetup { } // configureRuntimes does what needs to happen to get a runtime going. -func configureRuntimes(h *host.Host) { - runner, err := machine.CommandRunner(h) - if err != nil { - exit.WithError("Failed to get command runner", err) - } - +func configureRuntimes(h *host.Host, runner bootstrapper.CommandRunner) cruntime.Manager { config := cruntime.Config{Type: viper.GetString(containerRuntime), Runner: runner} cr, err := cruntime.New(config) if err != nil { @@ -469,7 +469,7 @@ func configureRuntimes(h *host.Host) { if err != nil { exit.WithError("Failed to enable container runtime", err) } - + return cr } // waitCacheImages blocks until the image cache jobs complete @@ -484,7 +484,7 @@ func waitCacheImages(g *errgroup.Group) { } // bootstrapCluster starts Kubernetes using the chosen bootstrapper -func bootstrapCluster(bs bootstrapper.Bootstrapper, kc cfg.KubernetesConfig, preexisting bool) { +func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner bootstrapper.CommandRunner, kc cfg.KubernetesConfig, preexisting bool) { console.OutStyle("pulling", "Pulling images used by Kubernetes %s ...", kc.KubernetesVersion) if err := bs.PullImages(kc); err != nil { console.OutStyle("failure", "Unable to pull images, which may be OK: %v", err) @@ -495,19 +495,19 @@ func bootstrapCluster(bs bootstrapper.Bootstrapper, kc cfg.KubernetesConfig, pre if preexisting { console.OutStyle("restarting", "Relaunching Kubernetes %s using %s ... ", kc.KubernetesVersion, bsName) if err := bs.RestartCluster(kc); err != nil { - exit.WithError("Error restarting cluster", err) + exit.WithProblems("Error restarting cluster", err, logs.FindProblems(r, bs, runner)) } return } console.OutStyle("launch", "Launching Kubernetes %s using %s ... ", kc.KubernetesVersion, bsName) if err := bs.StartCluster(kc); err != nil { - exit.WithError("Error starting cluster", err) + exit.WithProblems("Error starting cluster", err, logs.FindProblems(r, bs, runner)) } } // validateCluster validates that the cluster is well-configured and healthy -func validateCluster(bs bootstrapper.Bootstrapper, ip string) { +func validateCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner bootstrapper.CommandRunner, ip string) { console.OutStyle("verifying-noline", "Verifying component health ...") kStat := func() (err error) { st, err := bs.GetKubeletStatus() @@ -519,7 +519,7 @@ func validateCluster(bs bootstrapper.Bootstrapper, ip string) { } err := pkgutil.RetryAfter(20, kStat, 3*time.Second) if err != nil { - exit.WithError("kubelet checks failed", err) + exit.WithProblems("kubelet checks failed", err, logs.FindProblems(r, bs, runner)) } aStat := func() (err error) { st, err := bs.GetApiServerStatus(net.ParseIP(ip)) @@ -532,7 +532,7 @@ func validateCluster(bs bootstrapper.Bootstrapper, ip string) { err = pkgutil.RetryAfter(30, aStat, 10*time.Second) if err != nil { - exit.WithError("apiserver checks failed", err) + exit.WithProblems("apiserver checks failed", err, logs.FindProblems(r, bs, runner)) } console.OutLn("") } diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index 6cd961b18d45..4140f58548a9 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -17,13 +17,20 @@ limitations under the License. package bootstrapper import ( - "io" "net" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" ) +// LogOptions are options to be passed to LogCommands +type LogOptions struct { + // Lines is the number of recent log lines to include, as in tail -n. + Lines int + // Follow is whether or not to actively follow the logs, as in tail -f. + Follow bool +} + // Bootstrapper contains all the methods needed to bootstrap a kubernetes cluster type Bootstrapper interface { // PullImages pulls images necessary for a cluster. Success should not be required. @@ -32,7 +39,8 @@ type Bootstrapper interface { UpdateCluster(config.KubernetesConfig) error RestartCluster(config.KubernetesConfig) error DeleteCluster(config.KubernetesConfig) error - GetClusterLogsTo(follow bool, out io.Writer) error + // LogCommands returns a map of log type to a command which will display that log. + LogCommands(LogOptions) map[string]string SetupCerts(cfg config.KubernetesConfig) error GetKubeletStatus() (string, error) GetApiServerStatus(net.IP) (string, error) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 1268eb108b9f..f5f4a8e3fcbf 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -21,7 +21,6 @@ import ( "crypto" "crypto/tls" "fmt" - "io" "net" "net/http" "os" @@ -121,28 +120,17 @@ func (k *KubeadmBootstrapper) GetApiServerStatus(ip net.IP) (string, error) { return state.Running.String(), nil } -// TODO(r2d4): Should this aggregate all the logs from the control plane? -// Maybe subcommands for each component? minikube logs apiserver? -func (k *KubeadmBootstrapper) GetClusterLogsTo(follow bool, out io.Writer) error { - var flags []string - if follow { - flags = append(flags, "-f") +// LogCommands returns a map of log type to a command which will display that log. +func (k *KubeadmBootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string { + var kcmd strings.Builder + kcmd.WriteString("journalctl -u kubelet") + if o.Lines > 0 { + kcmd.WriteString(fmt.Sprintf(" -n %d", o.Lines)) } - logsCommand := fmt.Sprintf("sudo journalctl %s -u kubelet", strings.Join(flags, " ")) - - if follow { - if err := k.c.CombinedOutputTo(logsCommand, out); err != nil { - return errors.Wrap(err, "getting cluster logs") - } - } else { - - logs, err := k.c.CombinedOutput(logsCommand) - if err != nil { - return errors.Wrap(err, "getting cluster logs") - } - fmt.Fprint(out, logs) + if o.Follow { + kcmd.WriteString(" -f") } - return nil + return map[string]string{"kubelet": kcmd.String()} } func (k *KubeadmBootstrapper) StartCluster(k8s config.KubernetesConfig) error { diff --git a/pkg/minikube/console/style.go b/pkg/minikube/console/style.go index 1265a6558070..6c84a51fae28 100644 --- a/pkg/minikube/console/style.go +++ b/pkg/minikube/console/style.go @@ -40,8 +40,6 @@ var styles = map[string]style{ "fatal": {Prefix: "πŸ’£ "}, "notice": {Prefix: "πŸ“Œ "}, "ready": {Prefix: "πŸ„ "}, - "running": {Prefix: "πŸƒ "}, - "provisioning": {Prefix: "🌱 "}, "restarting": {Prefix: "πŸ”„ "}, "stopping": {Prefix: "βœ‹ "}, "stopped": {Prefix: "πŸ›‘ "}, @@ -49,11 +47,14 @@ var styles = map[string]style{ "waiting": {Prefix: "βŒ› "}, "usage": {Prefix: "πŸ’‘ "}, "launch": {Prefix: "πŸš€ "}, - "sad": {Prefix: "😿 "}, "thumbs-up": {Prefix: "πŸ‘ "}, "option": {Prefix: " β–ͺ "}, // Indented bullet - "url": {Prefix: "πŸ‘‰ "}, + "log-entry": {Prefix: " "}, // Indent "crushed": {Prefix: "πŸ’” "}, + "running": {Prefix: "πŸƒ "}, + "provisioning": {Prefix: "🌱 "}, + "sad": {Prefix: "😿 "}, + "url": {Prefix: "πŸ‘‰ "}, // Specialized purpose styles "iso-download": {Prefix: "πŸ’Ώ "}, diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 611197a3b598..e46a30e5daa9 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -107,3 +107,8 @@ func (r *Containerd) KillContainers(ids []string) error { func (r *Containerd) StopContainers(ids []string) error { return stopCRIContainers(r.Runner, ids) } + +// ContainerLogCmd returns the command to retrieve the log for a container based on ID +func (r *Containerd) ContainerLogCmd(id string, len int, follow bool) string { + return criContainerLogCmd(id, len, follow) +} diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index 67f4595ef0dd..0ccc7bfa57e3 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -60,3 +60,18 @@ image-endpoint: unix://{{.Socket}} } return cr.Run(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(cPath), b.String(), cPath)) } + +// criContainerLogCmd returns the command to retrieve the log for a container based on ID +func criContainerLogCmd(id string, len int, follow bool) string { + var cmd strings.Builder + cmd.WriteString("crictl logs ") + if len > 0 { + cmd.WriteString(fmt.Sprintf("--tail %d ", len)) + } + if follow { + cmd.WriteString("--follow ") + } + + cmd.WriteString(id) + return cmd.String() +} diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 2897dd2c06d5..93aa426ddf3d 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -106,3 +106,8 @@ func (r *CRIO) KillContainers(ids []string) error { func (r *CRIO) StopContainers(ids []string) error { return stopCRIContainers(r.Runner, ids) } + +// ContainerLogCmd returns the command to retrieve the log for a container based on ID +func (r *CRIO) ContainerLogCmd(id string, len int, follow bool) string { + return criContainerLogCmd(id, len, follow) +} diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 9f478086164d..fe670c29e517 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -61,6 +61,8 @@ type Manager interface { KillContainers([]string) error // StopContainers stops containers based on ID StopContainers([]string) error + // ContainerLogCmd returns the command to retrieve the log for a container based on ID + ContainerLogCmd(string, int, bool) string } // Config is runtime configuration diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 92f4c0bb5485..f70a513603da 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -101,3 +101,18 @@ func (r *Docker) KillContainers(ids []string) error { func (r *Docker) StopContainers(ids []string) error { return r.Runner.Run(fmt.Sprintf("docker stop %s", strings.Join(ids, " "))) } + +// ContainerLogCmd returns the command to retrieve the log for a container based on ID +func (r *Docker) ContainerLogCmd(id string, len int, follow bool) string { + var cmd strings.Builder + cmd.WriteString("docker logs ") + if len > 0 { + cmd.WriteString(fmt.Sprintf("--tail %d ", len)) + } + if follow { + cmd.WriteString("--follow ") + } + + cmd.WriteString(id) + return cmd.String() +} diff --git a/pkg/minikube/exit/exit.go b/pkg/minikube/exit/exit.go index f5a1d9ea19f5..fad5b18df386 100644 --- a/pkg/minikube/exit/exit.go +++ b/pkg/minikube/exit/exit.go @@ -18,6 +18,7 @@ limitations under the License. package exit import ( + "fmt" "os" "github.com/golang/glog" @@ -35,6 +36,9 @@ const ( IO = 74 // IO represents an I/O error Config = 78 // Config represents an unconfigured or misconΒ­figured state Permissions = 77 // Permissions represents a permissions error + + // MaxProblems controls the number of problems to show for each source + MaxProblems = 3 ) // Usage outputs a usage error and exits with error code 64 @@ -53,14 +57,34 @@ func WithCode(code int, format string, a ...interface{}) { // WithError outputs an error and exits. func WithError(msg string, err error) { - console.Fatal(msg+": %v", err) - console.Err("\n") - console.ErrStyle("sad", "Sorry that minikube crashed. If this was unexpected, we would love to hear from you:") - console.ErrStyle("url", "https://github.com/kubernetes/minikube/issues/new") - // use Warning because Error will display a duplicate message to stderr - glog.Warningf(msg) + displayError(msg, err) // Here is where we would insert code to optionally upload a stack trace. // We can be smarter about guessing exit codes, but EX_SOFTWARE should suffice. os.Exit(Software) } + +// WithProblems outputs an error along with any autodetected problems, and exits. +func WithProblems(msg string, err error, problems map[string][]string) { + displayError(msg, err) + + for name, lines := range problems { + console.OutStyle("failure", "Problems detected in %q:", name) + if len(lines) > MaxProblems { + lines = lines[:MaxProblems] + } + for _, l := range lines { + console.OutStyle("log-entry", l) + } + } + os.Exit(Software) +} + +func displayError(msg string, err error) { + // use Warning because Error will display a duplicate message to stderr + glog.Warningf(fmt.Sprintf("%s: %v", msg, err)) + console.Fatal(msg+": %v", err) + console.Err("\n") + console.ErrStyle("sad", "Sorry that minikube crashed. If this was unexpected, we would love to hear from you:") + console.ErrStyle("url", "https://github.com/kubernetes/minikube/issues/new") +} diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go new file mode 100644 index 000000000000..11f63e90137d --- /dev/null +++ b/pkg/minikube/logs/logs.go @@ -0,0 +1,152 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// package logs are convenience methods for fetching logs from a minikube cluster +package logs + +import ( + "bufio" + "bytes" + "fmt" + "os" + "regexp" + "sort" + "strings" + + "github.com/golang/glog" + "k8s.io/minikube/pkg/minikube/bootstrapper" + "k8s.io/minikube/pkg/minikube/console" + "k8s.io/minikube/pkg/minikube/cruntime" +) + +// rootCauseRe is a regular expression that matches known failure root causes +var rootCauseRe = regexp.MustCompile(`^error: |eviction manager: pods.* evicted|unknown flag: --`) + +// importantPods are a list of pods to retrieve logs for, in addition to the bootstrapper logs. +var importantPods = []string{ + "k8s_kube-apiserver", + "k8s_coredns_coredns", + "k8s_kube-scheduler", +} + +// lookbackwardsCount is how far back to look in a log for problems. This should be large enough to +// include usage messages from a failed binary, but small enough to not include irrelevant problems. +const lookBackwardsCount = 200 + +// Follow follows logs from multiple files in tail(1) format +func Follow(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner) error { + cs := []string{} + for _, v := range logCommands(r, bs, 0, true) { + cs = append(cs, v+" &") + } + cs = append(cs, "wait") + return runner.CombinedOutputTo(strings.Join(cs, " "), os.Stdout) +} + +// IsProblem returns whether this line matches a known problem +func IsProblem(line string) bool { + return rootCauseRe.MatchString(line) +} + +// FindProblems finds possible root causes among the logs +func FindProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner) map[string][]string { + pMap := map[string][]string{} + cmds := logCommands(r, bs, lookBackwardsCount, false) + for name, cmd := range cmds { + glog.Infof("Gathering logs for %s ...", name) + var b bytes.Buffer + err := runner.CombinedOutputTo(cmds[name], &b) + if err != nil { + glog.Warningf("failed %s: %s: %v", name, cmd, err) + continue + } + scanner := bufio.NewScanner(&b) + problems := []string{} + for scanner.Scan() { + l := scanner.Text() + if IsProblem(l) { + glog.Warningf("Found %s problem: %s", name, l) + problems = append(problems, l) + } + } + if len(problems) > 0 { + pMap[name] = problems + } + } + return pMap +} + +// OutputProblems outputs discovered problems. +func OutputProblems(problems map[string][]string, maxLines int) { + for name, lines := range problems { + console.OutStyle("failure", "Problems detected in %q:", name) + if len(lines) > maxLines { + lines = lines[len(lines)-maxLines:] + } + for _, l := range lines { + console.OutStyle("log-entry", l) + } + } +} + +// Output displays logs from multiple sources in tail(1) format +func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner, lines int) error { + cmds := logCommands(r, bs, lines, false) + names := []string{} + for k := range cmds { + names = append(names, k) + } + sort.Strings(names) + + failed := []string{} + for _, name := range names { + console.OutLn("==> %s <==", name) + var b bytes.Buffer + err := runner.CombinedOutputTo(cmds[name], &b) + if err != nil { + glog.Errorf("failed: %v", err) + failed = append(failed, name) + continue + } + scanner := bufio.NewScanner(&b) + for scanner.Scan() { + console.OutLn(scanner.Text()) + } + } + if len(failed) > 0 { + return fmt.Errorf("unable to fetch logs for: %s", strings.Join(failed, ", ")) + } + return nil +} + +// logCommands returns a list of commands that would be run to receive the anticipated logs +func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, length int, follow bool) map[string]string { + cmds := bs.LogCommands(bootstrapper.LogOptions{Lines: length, Follow: follow}) + for _, pod := range importantPods { + ids, err := r.ListContainers(pod) + if err != nil { + glog.Errorf("Failed to list containers for %q: %v", pod, err) + continue + } + glog.Infof("%d containers: %s", len(ids), ids) + if len(ids) == 0 { + cmds[pod] = fmt.Sprintf("No container was found matching %q", pod) + continue + } + cmds[pod] = r.ContainerLogCmd(ids[0], length, follow) + } + return cmds +} diff --git a/pkg/minikube/logs/logs_test.go b/pkg/minikube/logs/logs_test.go new file mode 100644 index 000000000000..245f5aa4d384 --- /dev/null +++ b/pkg/minikube/logs/logs_test.go @@ -0,0 +1,44 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logs + +import ( + "testing" +) + +func TestIsProblem(t *testing.T) { + var tests = []struct { + name string + want bool + input string + }{ + {"almost", false, "F2350 I would love to be an unknown flag, but I am not -- :( --"}, + {"apiserver-required-flag #1962", true, "error: [service-account-issuer is a required flag when BoundServiceAccountTokenVolume is enabled, --service-account-signing-key-file and --service-account-issuer are required flags"}, + {"kubelet-eviction #", true, "I0213 07:16:44.041623 2410 eviction_manager.go:187] eviction manager: pods kube-apiserver-minikube_kube-system(87f41e2e0629c3deb5c2239e08d8045d) evicted, waiting for pod to be cleaned up"}, + {"kubelet-unknown-flag #3655", true, "F0212 14:55:46.443031 2693 server.go:148] unknown flag: --AllowedUnsafeSysctls"}, + {"apiserver-auth-mode #2852", true, `{"log":"Error: unknown flag: --Authorization.Mode\n","stream":"stderr","time":"2018-06-17T22:16:35.134161966Z"}`}, + {"apiserver-admission #3524", true, "error: unknown flag: --GenericServerRunOptions.AdmissionControl"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := IsProblem(tc.input) + if got != tc.want { + t.Fatalf("IsProblem(%s)=%v, want %v", tc.input, got, tc.want) + } + }) + } +} diff --git a/pkg/util/kubernetes.go b/pkg/util/kubernetes.go index 927cf1dd7835..0b8df0d0f787 100644 --- a/pkg/util/kubernetes.go +++ b/pkg/util/kubernetes.go @@ -20,8 +20,11 @@ import ( "fmt" "time" + "github.com/golang/glog" "github.com/pkg/errors" - + appsv1 "k8s.io/api/apps/v1" + "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" @@ -29,16 +32,15 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - - appsv1 "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/kubernetes" - - "github.com/golang/glog" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" +) + +var ( + ReasonableMutateTime = time.Minute * 1 + ReasonableStartTime = time.Minute * 5 ) type PodStore struct { @@ -108,11 +110,11 @@ func StartPods(c kubernetes.Interface, namespace string, pod v1.Pod, waitForRunn return nil } -// Wait up to 10 minutes for all matching pods to become Running and at least one -// matching pod exists. +// WaitForPodsWithLabelRunning waits for all matching pods to become Running and at least one matching pod exists. func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels.Selector) error { + glog.Infof("Waiting for pod with label %q in ns %q ...", ns, label) lastKnownPodNumber := -1 - return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) { + return wait.PollImmediate(constants.APICallRetryInterval, ReasonableStartTime, func() (bool, error) { listOpts := metav1.ListOptions{LabelSelector: label.String()} pods, err := c.CoreV1().Pods(ns).List(listOpts) if err != nil { @@ -139,9 +141,9 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels }) } -// Wait up to 10 minutes for a pod to be deleted +// WaitForPodDelete waits for a pod to be deleted func WaitForPodDelete(c kubernetes.Interface, ns string, label labels.Selector) error { - return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) { + return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) { listOpts := metav1.ListOptions{LabelSelector: label.String()} pods, err := c.CoreV1().Pods(ns).List(listOpts) if err != nil { @@ -152,9 +154,9 @@ func WaitForPodDelete(c kubernetes.Interface, ns string, label labels.Selector) }) } -// Wait up to 10 minutes for the given event to appear +// WaitForEvent waits for the given event to appear func WaitForEvent(c kubernetes.Interface, ns string, reason string) error { - return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) { + return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) { events, err := c.Events().Events("default").List(metav1.ListOptions{}) if err != nil { glog.Infof("error getting events: %v", err)