cmd: diagnose problems downloading release image

cmd/openshift-install/analyze.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/openshift/installer/pkg/gather/service"
+)
+
+var (
+	analyzeOpts struct {
+		gatherBundle string
+	}
+)
+
+func newAnalyzeCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "analyze",
+		Short: "Analyze debugging data for a given installation failure",
+		Long: `Analyze debugging data for a given installation failure.
+
+This command helps users to analyze the reasons for an installation that failed while bootstrapping.`,
+		Args: cobra.ExactArgs(0),
+		Run: func(_ *cobra.Command, _ []string) {
+			gatherBundle := analyzeOpts.gatherBundle
+			if gatherBundle == "" {
+				var err error
+				gatherBundle, err = getGatherBundleFromAssetsDirectory()
+				if err != nil {
+					logrus.Fatal(err)
+				}
+			}
+			if !filepath.IsAbs(gatherBundle) {
+				gatherBundle = filepath.Join(rootOpts.dir, gatherBundle)
+			}
+			if err := service.AnalyzeGatherBundle(gatherBundle); err != nil {
+				logrus.Fatal(err)
+			}
+		},
+	}
+	cmd.PersistentFlags().StringVar(&analyzeOpts.gatherBundle, "file", "", "Filename of the bootstrap gather bundle; either absolute or relative to the assets directory")
+	return cmd
+}
+
+func getGatherBundleFromAssetsDirectory() (string, error) {
+	matches, err := filepath.Glob(filepath.Join(rootOpts.dir, "log-bundle-*.tar.gz"))
+	if err != nil {
+		return "", errors.Wrap(err, "could not find gather bundles in assets directory")
+	}
+	switch len(matches) {
+	case 0:
+		return "", errors.New("no bootstrap gather bundles found in assets directory")
+	case 1:
+		return matches[0], nil
+	default:
+		return "", errors.New("multiple bootstrap gather bundles found in assets directory; select specific gather bundle by using the --file flag")
+	}
+}

cmd/openshift-install/create.go

@@ -32,6 +32,7 @@ import (
 	assetstore "github.com/openshift/installer/pkg/asset/store"
 	targetassets "github.com/openshift/installer/pkg/asset/targets"
 	destroybootstrap "github.com/openshift/installer/pkg/destroy/bootstrap"
+	"github.com/openshift/installer/pkg/gather/service"
 	timer "github.com/openshift/installer/pkg/metrics/timer"
 	"github.com/openshift/installer/pkg/types/baremetal"
 	cov1helpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers"
@@ -115,11 +116,15 @@ var (
 					if err2 := logClusterOperatorConditions(ctx, config); err2 != nil {
 						logrus.Error("Attempted to gather ClusterOperator status after installation failure: ", err2)
 					}
-					if err2 := runGatherBootstrapCmd(rootOpts.dir); err2 != nil {
+					bundlePath, err2 := runGatherBootstrapCmd(rootOpts.dir)
+					if err2 != nil {
 						logrus.Error("Attempted to gather debug logs after installation failure: ", err2)
 					}
 					logrus.Error("Bootstrap failed to complete: ", err.Unwrap())
 					logrus.Error(err.Error())
+					if err2 := service.AnalyzeGatherBundle(bundlePath); err2 != nil {
+						logrus.Error("Attempted to analyze the debug logs after installation failure: ", err2)
+					}
 					logrus.Fatal("Bootstrap failed to complete")
 				}
 				timer.StopTimer("Bootstrap Complete")

cmd/openshift-install/gather.go

@@ -23,6 +23,7 @@ import (
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	assetstore "github.com/openshift/installer/pkg/asset/store"
 	"github.com/openshift/installer/pkg/asset/tls"
+	"github.com/openshift/installer/pkg/gather/service"
 	"github.com/openshift/installer/pkg/gather/ssh"
 	"github.com/openshift/installer/pkg/terraform"
 	gatheraws "github.com/openshift/installer/pkg/terraform/gather/aws"
@@ -52,7 +53,7 @@ func newGatherCmd() *cobra.Command {
 		Short: "Gather debugging data for a given installation failure",
 		Long: `Gather debugging data for a given installation failure.
 
-When installation for Openshift cluster fails, gathering all the data useful for debugging can
+When installation for OpenShift cluster fails, gathering all the data useful for debugging can
 become a difficult task. This command helps users to collect the most relevant information that can be used
 to debug the installation failures`,
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -65,9 +66,10 @@ to debug the installation failures`,
 
 var (
 	gatherBootstrapOpts struct {
-		bootstrap string
-		masters   []string
-		sshKeys   []string
+		bootstrap    string
+		masters      []string
+		sshKeys      []string
+		skipAnalysis bool
 	}
 )
 
@@ -79,38 +81,44 @@ func newGatherBootstrapCmd() *cobra.Command {
 		Run: func(_ *cobra.Command, _ []string) {
 			cleanup := setupFileHook(rootOpts.dir)
 			defer cleanup()
-			err := runGatherBootstrapCmd(rootOpts.dir)
+			bundlePath, err := runGatherBootstrapCmd(rootOpts.dir)
 			if err != nil {
 				logrus.Fatal(err)
 			}
+			if !gatherBootstrapOpts.skipAnalysis {
+				if err := service.AnalyzeGatherBundle(bundlePath); err != nil {
+					logrus.Fatal(err)
+				}
+			}
 		},
 	}
 	cmd.PersistentFlags().StringVar(&gatherBootstrapOpts.bootstrap, "bootstrap", "", "Hostname or IP of the bootstrap host")
 	cmd.PersistentFlags().StringArrayVar(&gatherBootstrapOpts.masters, "master", []string{}, "Hostnames or IPs of all control plane hosts")
 	cmd.PersistentFlags().StringArrayVar(&gatherBootstrapOpts.sshKeys, "key", []string{}, "Path to SSH private keys that should be used for authentication. If no key was provided, SSH private keys from user's environment will be used")
+	cmd.PersistentFlags().BoolVar(&gatherBootstrapOpts.skipAnalysis, "skipAnalysis", false, "Skip analysis of the gathered data")
 	return cmd
 }
 
-func runGatherBootstrapCmd(directory string) error {
+func runGatherBootstrapCmd(directory string) (string, error) {
 	assetStore, err := assetstore.NewStore(directory)
 	if err != nil {
-		return errors.Wrap(err, "failed to create asset store")
+		return "", errors.Wrap(err, "failed to create asset store")
 	}
 	// add the default bootstrap key pair to the sshKeys list
 	bootstrapSSHKeyPair := &tls.BootstrapSSHKeyPair{}
 	if err := assetStore.Fetch(bootstrapSSHKeyPair); err != nil {
-		return errors.Wrapf(err, "failed to fetch %s", bootstrapSSHKeyPair.Name())
+		return "", errors.Wrapf(err, "failed to fetch %s", bootstrapSSHKeyPair.Name())
 	}
 	tmpfile, err := ioutil.TempFile("", "bootstrap-ssh")
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer os.Remove(tmpfile.Name())
 	if _, err := tmpfile.Write(bootstrapSSHKeyPair.Private()); err != nil {
-		return err
+		return "", err
 	}
 	if err := tmpfile.Close(); err != nil {
-		return err
+		return "", err
 	}
 	gatherBootstrapOpts.sshKeys = append(gatherBootstrapOpts.sshKeys, tmpfile.Name())
 
@@ -120,54 +128,54 @@ func runGatherBootstrapCmd(directory string) error {
 		return unSupportedPlatformGather(directory)
 	}
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	config := &installconfig.InstallConfig{}
 	if err := assetStore.Fetch(config); err != nil {
-		return errors.Wrapf(err, "failed to fetch %s", config.Name())
+		return "", errors.Wrapf(err, "failed to fetch %s", config.Name())
 	}
 
 	tfstate, err := terraform.ReadState(tfStateFilePath)
 	if err != nil {
-		return errors.Wrapf(err, "failed to read state from %q", tfStateFilePath)
+		return "", errors.Wrapf(err, "failed to read state from %q", tfStateFilePath)
 	}
 	bootstrap, port, masters, err := extractHostAddresses(config.Config, tfstate)
 	if err != nil {
 		if err2, ok := err.(errUnSupportedGatherPlatform); ok {
 			logrus.Error(err2)
 			return unSupportedPlatformGather(directory)
 		}
-		return errors.Wrapf(err, "failed to get bootstrap and control plane host addresses from %q", tfStateFilePath)
+		return "", errors.Wrapf(err, "failed to get bootstrap and control plane host addresses from %q", tfStateFilePath)
 	}
 
 	return logGatherBootstrap(bootstrap, port, masters, directory)
 }
 
-func logGatherBootstrap(bootstrap string, port int, masters []string, directory string) error {
+func logGatherBootstrap(bootstrap string, port int, masters []string, directory string) (string, error) {
 	logrus.Info("Pulling debug logs from the bootstrap machine")
 	client, err := ssh.NewClient("core", net.JoinHostPort(bootstrap, strconv.Itoa(port)), gatherBootstrapOpts.sshKeys)
 	if err != nil {
 		if errors.Is(err, syscall.ECONNREFUSED) {
-			return errors.Wrap(err, "failed to connect to the bootstrap machine")
+			return "", errors.Wrap(err, "failed to connect to the bootstrap machine")
 		}
-		return errors.Wrap(err, "failed to create SSH client")
+		return "", errors.Wrap(err, "failed to create SSH client")
 	}
 
 	gatherID := time.Now().Format("20060102150405")
 	if err := ssh.Run(client, fmt.Sprintf("/usr/local/bin/installer-gather.sh --id %s %s", gatherID, strings.Join(masters, " "))); err != nil {
-		return errors.Wrap(err, "failed to run remote command")
+		return "", errors.Wrap(err, "failed to run remote command")
 	}
 	file := filepath.Join(directory, fmt.Sprintf("log-bundle-%s.tar.gz", gatherID))
 	if err := ssh.PullFileTo(client, fmt.Sprintf("/home/core/log-bundle-%s.tar.gz", gatherID), file); err != nil {
-		return errors.Wrap(err, "failed to pull log file from remote")
+		return "", errors.Wrap(err, "failed to pull log file from remote")
 	}
 	path, err := filepath.Abs(file)
 	if err != nil {
-		return errors.Wrap(err, "failed to stat log file")
+		return "", errors.Wrap(err, "failed to stat log file")
 	}
 	logrus.Infof("Bootstrap gather logs captured here %q", path)
-	return nil
+	return path, nil
 }
 
 func extractHostAddresses(config *types.InstallConfig, tfstate *terraform.State) (bootstrap string, port int, masters []string, err error) {
@@ -266,9 +274,9 @@ func (e errUnSupportedGatherPlatform) Error() string {
 	return e.Message
 }
 
-func unSupportedPlatformGather(directory string) error {
+func unSupportedPlatformGather(directory string) (string, error) {
 	if gatherBootstrapOpts.bootstrap == "" || len(gatherBootstrapOpts.masters) == 0 {
-		return errors.New("bootstrap host address and at least one control plane host address must be provided")
+		return "", errors.New("bootstrap host address and at least one control plane host address must be provided")
 	}
 
 	return logGatherBootstrap(gatherBootstrapOpts.bootstrap, 22, gatherBootstrapOpts.masters, directory)

cmd/openshift-install/main.go

@@ -58,6 +58,7 @@ func installerMain() {
 		newDestroyCmd(),
 		newWaitForCmd(),
 		newGatherCmd(),
+		newAnalyzeCmd(),
 		newVersionCmd(),
 		newGraphCmd(),
 		newCoreOSCmd(),