cmd: gather the logs from bootstrap instead of printing commands

Gopkg.toml

@@ -103,3 +103,7 @@ required = [
   branch = "master"
   name = "sigs.k8s.io/cluster-api-provider-azure"
   source = "https://github.com/openshift/cluster-api-provider-azure.git"
+
+[[constraint]]
+  name = "github.com/pkg/sftp"
+  version = "1.10.0"

cmd/openshift-install/gather.go

@@ -5,13 +5,15 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	assetstore "github.com/openshift/installer/pkg/asset/store"
+	"github.com/openshift/installer/pkg/gather/ssh"
 	"github.com/openshift/installer/pkg/terraform"
 	gatheraws "github.com/openshift/installer/pkg/terraform/gather/aws"
 	gatherazure "github.com/openshift/installer/pkg/terraform/gather/azure"
@@ -45,6 +47,7 @@ var (
 	gatherBootstrapOpts struct {
 		bootstrap string
 		masters   []string
+		sshKeys   []string
 	}
 )
 
@@ -64,14 +67,15 @@ func newGatherBootstrapCmd() *cobra.Command {
 	}
 	cmd.PersistentFlags().StringVar(&gatherBootstrapOpts.bootstrap, "bootstrap", "", "Hostname or IP of the bootstrap host")
 	cmd.PersistentFlags().StringArrayVar(&gatherBootstrapOpts.masters, "master", []string{}, "Hostnames or IPs of all control plane hosts")
+	cmd.PersistentFlags().StringArrayVar(&gatherBootstrapOpts.sshKeys, "key", []string{}, "Path to SSH private keys that should be used for authentication. If no key was provided, SSH private keys from user's environment will be used")
 	return cmd
 }
 
 func runGatherBootstrapCmd(directory string) error {
 	tfStateFilePath := filepath.Join(directory, terraform.StateFileName)
 	_, err := os.Stat(tfStateFilePath)
 	if os.IsNotExist(err) {
-		return unSupportedPlatformGather()
+		return unSupportedPlatformGather(directory)
 	}
 	if err != nil {
 		return err
@@ -95,30 +99,29 @@ func runGatherBootstrapCmd(directory string) error {
 	if err != nil {
 		if err2, ok := err.(errUnSupportedGatherPlatform); ok {
 			logrus.Error(err2)
-			return unSupportedPlatformGather()
+			return unSupportedPlatformGather(directory)
 		}
 		return errors.Wrapf(err, "failed to get bootstrap and control plane host addresses from %q", tfStateFilePath)
 	}
 
-	logGatherBootstrap(bootstrap, port, masters)
-	return nil
+	return logGatherBootstrap(bootstrap, port, masters, directory)
 }
 
-func logGatherBootstrap(bootstrap string, port int, masters []string) {
-	var (
-		sshport string
-		scpport string
-	)
-	if port != 22 {
-		sshport = fmt.Sprintf(" -p %d", port)
-		scpport = fmt.Sprintf(" -P %d", port)
+func logGatherBootstrap(bootstrap string, port int, masters []string, directory string) error {
+	logrus.Info("Pulling debug logs from the bootstrap machine")
+	client, err := ssh.NewClient("core", fmt.Sprintf("%s:%d", bootstrap, port), gatherBootstrapOpts.sshKeys)
+	if err != nil {
+		return errors.Wrap(err, "failed to create SSH client")
+	}
+	if err := ssh.Run(client, fmt.Sprintf("/usr/local/bin/installer-gather.sh %s", strings.Join(masters, " "))); err != nil {
+		return errors.Wrap(err, "failed to run remote command")
 	}
-	if s, ok := os.LookupEnv("SSH_AUTH_SOCK"); !ok || s == "" {
-		logrus.Info("Make sure ssh-agent is running, env SSH_AUTH_SOCK is set to the ssh-agent's UNIX socket and your private key is added to the agent.")
+	file := filepath.Join(directory, fmt.Sprintf("log-bundle-%s.tar.gz", time.Now().Format("20060102150405")))
+	if err := ssh.PullFileTo(client, "/home/core/log-bundle.tar.gz", file); err != nil {
+		return errors.Wrap(err, "failed to pull log file from remote")
 	}
-	logrus.Info("Use the following commands to gather logs from the cluster")
-	logrus.Infof("ssh -A%s core@%s '/usr/local/bin/installer-gather.sh %s'", sshport, bootstrap, strings.Join(masters, " "))
-	logrus.Infof("scp%s core@%s:~/log-bundle.tar.gz .", scpport, bootstrap)
+	logrus.Infof("Bootstrap gather logs captured here %q", file)
+	return nil
 }
 
 func extractHostAddresses(config *types.InstallConfig, tfstate *terraform.State) (bootstrap string, port int, masters []string, err error) {
@@ -175,11 +178,10 @@ func (e errUnSupportedGatherPlatform) Error() string {
 	return e.Message
 }
 
-func unSupportedPlatformGather() error {
+func unSupportedPlatformGather(directory string) error {
 	if gatherBootstrapOpts.bootstrap == "" || len(gatherBootstrapOpts.masters) == 0 {
 		return errors.New("boostrap host address and at least one control plane host address must be provided")
 	}
 
-	logGatherBootstrap(gatherBootstrapOpts.bootstrap, 22, gatherBootstrapOpts.masters)
-	return nil
+	return logGatherBootstrap(gatherBootstrapOpts.bootstrap, 22, gatherBootstrapOpts.masters, directory)
 }

pkg/gather/ssh/agent.go

@@ -0,0 +1,39 @@
+package ssh
+
+import (
+	"github.com/pkg/errors"
+	"golang.org/x/crypto/ssh/agent"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+)
+
+// newAgent initializes an SSH Agent with the keys.
+// If no keys are provided, it loads all the keys from the user's environment.
+func newAgent(keyPaths []string) (agent.Agent, error) {
+	keys, err := loadKeys(keyPaths)
+	if err != nil {
+		return nil, err
+	}
+	if len(keys) == 0 {
+		return nil, errors.New("no keys found for SSH agent")
+	}
+
+	ag := agent.NewKeyring()
+	var errs []error
+	for idx := range keys {
+		if err := ag.Add(agent.AddedKey{PrivateKey: keys[idx]}); err != nil {
+			errs = append(errs, errors.Wrap(err, "failed to add key to agent"))
+		}
+	}
+	if agg := utilerrors.NewAggregate(errs); agg != nil {
+		return nil, agg
+	}
+	return ag, nil
+}
+
+func loadKeys(paths []string) ([]interface{}, error) {
+	if len(paths) > 0 {
+		return LoadPrivateSSHKeys(paths)
+	}
+	return defaultPrivateSSHKeys()
+}

pkg/gather/ssh/ssh.go

@@ -0,0 +1,136 @@
+// Package ssh contains utilities that help gather logs, etc. on failures using ssh.
+package ssh
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/openshift/installer/pkg/lineprinter"
+	"github.com/pkg/errors"
+	"github.com/pkg/sftp"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/agent"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+)
+
+// NewClient creates a new SSH client which can be used to SSH to address using user and the keys.
+//
+// if keys list is empty, it tries to load the keys from the user's environment.
+func NewClient(user, address string, keys []string) (*ssh.Client, error) {
+	ag, err := newAgent(keys)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to initialize the SSH agent")
+	}
+
+	client, err := ssh.Dial("tcp", address, &ssh.ClientConfig{
+		User: user,
+		Auth: []ssh.AuthMethod{
+			// Use a callback rather than PublicKeys
+			// so we only consult the agent once the remote server
+			// wants it.
+			ssh.PublicKeysCallback(ag.Signers),
+		},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	})
+	if err != nil {
+		return nil, err
+	}
+	if err := agent.ForwardToAgent(client, ag); err != nil {
+		return nil, errors.Wrap(err, "failed to forward agent")
+	}
+	return client, nil
+}
+
+// Run uses an SSH client to execute commands.
+func Run(client *ssh.Client, command string) error {
+	sess, err := client.NewSession()
+	if err != nil {
+		return err
+	}
+	defer sess.Close()
+	if err := agent.RequestAgentForwarding(sess); err != nil {
+		return errors.Wrap(err, "failed to setup request agent forwarding")
+	}
+
+	debugW := &lineprinter.LinePrinter{Print: (&lineprinter.Trimmer{WrappedPrint: logrus.Debug}).Print}
+	defer debugW.Close()
+	sess.Stdout = debugW
+	sess.Stderr = debugW
+	return sess.Run(command)
+}
+
+// PullFileTo downloads the file from remote server using SSH connection and writes to localPath.
+func PullFileTo(client *ssh.Client, remotePath, localPath string) error {
+	sc, err := sftp.NewClient(client)
+	if err != nil {
+		return errors.Wrap(err, "failed to initialize the sftp client")
+	}
+	defer sc.Close()
+
+	// Open the source file
+	rFile, err := sc.Open(remotePath)
+	if err != nil {
+		return errors.Wrap(err, "failed to open remote file")
+	}
+	defer rFile.Close()
+
+	lFile, err := os.Create(localPath)
+	if err != nil {
+		return errors.Wrap(err, "failed to create file")
+	}
+	defer lFile.Close()
+
+	if _, err := rFile.WriteTo(lFile); err != nil {
+		return err
+	}
+	return nil
+}
+
+// defaultPrivateSSHKeys returns a list of all the PRIVATE SSH keys from user's home directory.
+// It does not return any intermediate errors if at least one private key was loaded.
+func defaultPrivateSSHKeys() ([]interface{}, error) {
+	d := filepath.Join(os.Getenv("HOME"), ".ssh")
+	paths, err := ioutil.ReadDir(d)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to read directory %q", d)
+	}
+
+	var files []string
+	for _, path := range paths {
+		if path.IsDir() {
+			continue
+		}
+		files = append(files, filepath.Join(d, path.Name()))
+	}
+	keys, err := LoadPrivateSSHKeys(files)
+	if keys != nil && len(keys) > 0 {
+		return keys, nil
+	}
+	return nil, err
+}
+
+// LoadPrivateSSHKeys try to optimistically load PRIVATE SSH keys from the all paths.
+func LoadPrivateSSHKeys(paths []string) ([]interface{}, error) {
+	var errs []error
+	var keys []interface{}
+	for _, path := range paths {
+		data, err := ioutil.ReadFile(path)
+		if err != nil {
+			errs = append(errs, errors.Wrapf(err, "failed to read %q", path))
+			continue
+		}
+		key, err := ssh.ParseRawPrivateKey(data)
+		if err != nil {
+			errs = append(errs, errors.Wrapf(err, "failed to parse SSH private key from %q", path))
+			continue
+		}
+		keys = append(keys, key)
+	}
+	if err := utilerrors.NewAggregate(errs); err != nil {
+		return keys, err
+	}
+	return keys, nil
+}