feat: Handling git credentials via credential helper

fixes jenkins-x#5772

pkg/cmd/cmd.go

@@ -79,7 +79,7 @@ func NewJXCommand(f clients.Factory, in terminal.FileReader, out terminal.FileWr
 	rootCommand := &cobra.Command{
 		Use:              "jx",
 		Short:            "jx is a command line tool for working with Jenkins X",
-		PersistentPreRun: setLoggingLevel,
+		PersistentPreRun: persistentPreRun,
 		Run:              runHelp,
 	}
 
@@ -304,7 +304,11 @@ func fullPath(command *cobra.Command) string {
 	return name
 }
 
-func setLoggingLevel(cmd *cobra.Command, args []string) {
+func persistentPreRun(cmd *cobra.Command, args []string) {
+	setLoggingLevel(cmd)
+}
+
+func setLoggingLevel(cmd *cobra.Command) {
 	verbose, err := strconv.ParseBool(cmd.Flag(opts.OptionVerbose).Value.String())
 	if err != nil {
 		log.Logger().Errorf("Unable to check if the verbose flag is set")

pkg/cmd/step/git/credentials/step_git_credentials.go

@@ -7,6 +7,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 
 	"github.com/jenkins-x/jx/pkg/auth"
 	"github.com/jenkins-x/jx/pkg/cmd/opts/step"
@@ -37,6 +38,7 @@ type StepGitCredentialsOptions struct {
 	GitHubAppOwner    string
 	GitKind           string
 	CredentialsSecret string
+	AskPass           bool
 }
 
 type credentials struct {
@@ -57,6 +59,9 @@ var (
 
 		# generate the Git credentials to a output file
 		jx step git credentials -o /tmp/mycreds
+
+		# respond to a GIT_ASKPASS request
+		jx step git credentials --ask-pass
 `)
 )
 
@@ -82,6 +87,7 @@ func NewCmdStepGitCredentials(commonOpts *opts.CommonOptions) *cobra.Command {
 	cmd.Flags().StringVarP(&options.GitHubAppOwner, optionGitHubAppOwner, "g", "", "The owner (organisation or user name) if using GitHub App based tokens")
 	cmd.Flags().StringVarP(&options.CredentialsSecret, "credentials-secret", "s", "", "The secret name to read the credentials from")
 	cmd.Flags().StringVarP(&options.GitKind, "git-kind", "", "", "The git kind. e.g. github, bitbucketserver etc")
+	cmd.Flags().BoolVar(&options.AskPass, "ask-pass", false, "respond to a GIT_ASKPASS request")
 	return cmd
 }
 
@@ -142,9 +148,22 @@ func (o *StepGitCredentialsOptions) Run() error {
 
 	credentials, err := o.CreateGitCredentialsFromAuthService(authConfigSvc)
 	if err != nil {
+
 		return errors.Wrap(err, "creating git credentials")
 	}
-	return o.createGitCredentialsFile(outFile, credentials)
+
+	if o.AskPass {
+		response := o.askPass(os.Args[len(os.Args)-1], credentials)
+		fmt.Println(response)
+		return nil
+	} else {
+		outFile, err := o.determineOutputFile()
+		if err != nil {
+			return err
+		}
+
+		return o.createGitCredentialsFile(outFile, credentials)
+	}
 }
 
 func (o *StepGitCredentialsOptions) GitCredentialsFileData(credentials []credentials) ([]byte, error) {
@@ -168,6 +187,36 @@ func (o *StepGitCredentialsOptions) GitCredentialsFileData(credentials []credent
 	return buffer.Bytes(), nil
 }
 
+// TODO issue-5772 handle input from GIT_ASKPASS properly by parsing the git provider
+func (o *StepGitCredentialsOptions) askPass(question string, credentials []credentials) string {
+	re, err := regexp.Compile(`^.*'(?P<url>http.*)':.*$`)
+	if err != nil {
+		log.Logger().Errorf("unable to compile regexp: %s", err.Error())
+		return ""
+	}
+
+	matches := re.FindStringSubmatch(question)
+
+	if matches == nil {
+		log.Logger().Errorf("unexpected password challenge format: %s", question)
+		return ""
+	}
+
+	gitURL, err := url.Parse(matches[1])
+	if err != nil {
+		log.Logger().Errorf("invalid git URL: %s", err.Error())
+		return ""
+	}
+
+	for _, creds := range credentials {
+		if gitURL.User.Username() == creds.user {
+			return creds.password
+		}
+	}
+
+	return ""
+}
+
 func (o *StepGitCredentialsOptions) determineOutputFile() (string, error) {
 	outFile := o.OutputFile
 	if outFile == "" {

pkg/cmd/step/verify/step_verify_environments.go

@@ -487,12 +487,8 @@ func (o *StepVerifyEnvironmentsOptions) pushDevEnvironmentUpdates(environmentRep
 		}
 	}
 
-	userDetails := provider.UserAuth()
-	authenticatedPushURL, err := gitter.CreateAuthenticatedURL(environmentRepo.CloneURL, &userDetails)
-	if err != nil {
-		return errors.Wrapf(err, "failed to create push URL for %s", environmentRepo.CloneURL)
-	}
-	err = gitter.Push(localRepoDir, authenticatedPushURL, true, "master")
+	// TODO issue-5772 determine final remote
+	err = gitter.Push(localRepoDir, "origin", true, "master")
 	if err != nil {
 		return errors.Wrapf(err, "unable to push %s to %s", localRepoDir, environmentRepo.URL)
 	}

pkg/gits/git_cli.go

@@ -24,6 +24,11 @@ import (
 
 const (
 	replaceInvalidBranchChars = '_'
+	askPassBin                = "git-askpass"
+	askPassNonWindows         = `#!/usr/bin/env sh
+
+jx step git credentials --ask-pass "$@"
+`  // #nosec
 )
 
 var (
@@ -43,9 +48,40 @@ func NewGitCLI() *GitCLI {
 	}
 	// Ensure that error output is in English so parsing work
 	cli.Env["LC_ALL"] = "C"
+	cli.Env["JX_LOG_LEVEL"] = "error"
+	cli.configureGitAskPass()
 	return cli
 }
 
+func (g *GitCLI) configureGitAskPass() {
+	// TODO issue-5772 add a batch file for windows
+	configDir, err := util.ConfigDir()
+	if err != nil {
+		log.Logger().Errorf("unable to determine current JX_HOME: %s", err.Error())
+	}
+
+	binPath := filepath.Join(configDir, "bin")
+	err = os.MkdirAll(binPath, os.ModePerm)
+	if err != nil {
+		log.Logger().Errorf("unable to create bin directory: %s", err.Error())
+	}
+	askPassBin := filepath.Join(binPath, askPassBin)
+	exists, err := util.FileExists(askPassBin)
+	if err != nil {
+		log.Logger().Errorf("unable to determine file stats for %s: %s", askPassBin, err.Error())
+	}
+
+	if !exists {
+		data := []byte(askPassNonWindows)
+		err := ioutil.WriteFile(askPassBin, data, 0750)
+		if err != nil {
+			log.Logger().Errorf("unable to write %s: %s", askPassBin, err.Error())
+		}
+	}
+
+	g.Env["GIT_ASKPASS"] = askPassBin
+}
+
 // FindGitConfigDir tries to find the `.git` directory either in the current directory or in parent directories
 func (g *GitCLI) FindGitConfigDir(dir string) (string, string, error) {
 	d := dir

pkg/gits/helpers.go

@@ -11,6 +11,7 @@ import (
 	"reflect"
 	"sort"
 	"strings"
+	"time"
 
 	"gopkg.in/src-d/go-git.v4/config"
 
@@ -432,6 +433,8 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin
 		return "", "", nil, nil, errors.Wrapf(err, "failed to parse gitter URL %s", gitURL)
 	}
 
+	log.Logger().Debugf("ForkAndPullRepo gitURL: %s dir: %s baseRef: %s branchName: %s forkName: %s", gitURL, dir, baseRef, branchName, forkName)
+
 	username := provider.CurrentUsername()
 	originalOrg := originalInfo.Organisation
 	originalRepo := originalInfo.Name
@@ -517,6 +520,7 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin
 			return "", "", nil, nil, errors.WithStack(err)
 		}
 	} else {
+		// TODO issue-5572 this will not stash everything
 		err = gitter.StashPush(dir)
 		stashed = true
 		if err != nil {
@@ -526,31 +530,34 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin
 
 	// The long form of "git clone" has the advantage of working fine on an existing git repo and avoids checking out master
 	// and then another branch
-	err = gitter.SetRemoteURL(dir, originRemote, originURL)
+	originURLWithUser, err := addUserToURL(originURL, username)
+	if err != nil {
+		return "", "", nil, nil, errors.Wrapf(err, "unable to add username to git url", originURL)
+	}
+	err = gitter.SetRemoteURL(dir, originRemote, originURLWithUser)
 	if err != nil {
 		return "", "", nil, nil, errors.Wrapf(err, "failed to set %s url to %s", originRemote, originURL)
 	}
 	if fork {
 		upstreamRemote = "upstream"
-		err := gitter.SetRemoteURL(dir, upstreamRemote, upstreamInfo.CloneURL)
+		upstreamURLWithUser, err := addUserToURL(upstreamInfo.CloneURL, username)
+		if err != nil {
+			return "", "", nil, nil, errors.Wrapf(err, "unable to add username to git url", upstreamInfo.CloneURL)
+		}
+		err = gitter.SetRemoteURL(dir, upstreamRemote, upstreamURLWithUser)
 		if err != nil {
 			return "", "", nil, nil, errors.Wrapf(err, "setting remote upstream %q in forked environment repo", gitURL)
 		}
 	}
 
-	userDetails := provider.UserAuth()
-	originFetchURL, err := gitter.CreateAuthenticatedURL(originURL, &userDetails)
-	if err != nil {
-		return "", "", nil, nil, errors.Wrapf(err, "failed to create authenticated fetch URL for %s", originURL)
-	}
 	branchNameUUID, err := uuid.NewV4()
 	if err != nil {
 		return "", "", nil, nil, errors.WithStack(err)
 	}
 	originFetchBranch := branchNameUUID.String()
 
 	var upstreamFetchBranch string
-	err = gitter.FetchBranch(dir, originFetchURL, fmt.Sprintf("%s:%s", branchName, originFetchBranch))
+	err = gitter.FetchBranch(dir, "origin", fmt.Sprintf("%s:%s", branchName, originFetchBranch))
 
 	if err != nil {
 		if IsCouldntFindRemoteRefError(err, branchName) { // We can safely ignore missing remote branches, as they just don't exist
@@ -561,21 +568,17 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin
 	}
 
 	if upstreamRemote != originRemote || baseRef != branchName {
-		upstreamFetchURL, err := gitter.CreateAuthenticatedURL(upstreamInfo.CloneURL, &userDetails)
 		branchNameUUID, err := uuid.NewV4()
 		if err != nil {
 			return "", "", nil, nil, errors.WithStack(err)
 		}
 		upstreamFetchBranch = branchNameUUID.String()
-		if err != nil {
-			return "", "", nil, nil, errors.Wrapf(err, "failed to create authenticated fetch URL for %s", upstreamInfo.CloneURL)
-		}
+
 		// We're going to start our work from baseRef on the upstream
-		err = gitter.FetchBranch(dir, upstreamFetchURL, fmt.Sprintf("%s:%s", baseRef, upstreamFetchBranch))
+		err = gitter.FetchBranch(dir, upstreamRemote, fmt.Sprintf("%s:%s", baseRef, upstreamFetchBranch))
 		if err != nil {
 			return "", "", nil, nil, errors.WithStack(err)
 		}
-
 	}
 
 	// Work out what branch to use and check it out
@@ -599,6 +602,10 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin
 			return "", "", nil, nil, errors.WithStack(err)
 		}
 	} else if dirExists {
+		// TODO issue-5772 tmp
+		log.Logger().Info("Sleep 5 minutes")
+		time.Sleep(5 * time.Minute)
+		log.Logger().Info("Resuming")
 		toCherryPick, err = gitter.GetCommitsNotOnAnyRemote(dir, localBranchName)
 		if err != nil {
 			return "", "", nil, nil, errors.WithStack(err)
@@ -616,7 +623,11 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin
 
 	// Merge in any local committed changes we found
 	if len(toCherryPick) > 0 {
-		log.Logger().Infof("Attempting to cherry pick commits that were on %s but not yet pushed", localBranchName)
+		var shas = make([]string, len(toCherryPick))
+		for _, commit := range toCherryPick {
+			shas = append(shas, commit.SHA)
+		}
+		log.Logger().Debugf("Attempting to cherry pick commits %s that were on %s but not yet pushed", shas, localBranchName)
 	}
 	for _, c := range toCherryPick {
 		err = gitter.CherryPick(dir, c.SHA)
@@ -658,6 +669,20 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin
 	return dir, baseRef, upstreamInfo, forkInfo, nil
 }
 
+func addUserToURL(gitURL string, user string) (string, error) {
+	u, err := url.Parse(gitURL)
+	if err != nil {
+		return "", errors.Wrapf(err, "invalid git URL: %s", gitURL)
+	}
+
+	if user != "" {
+		userInfo := url.User(user)
+		u.User = userInfo
+	}
+
+	return u.String(), nil
+}
+
 // A PullRequestFilter defines a filter for finding pull requests
 type PullRequestFilter struct {
 	Labels []string
@@ -805,6 +830,8 @@ func DuplicateGitRepoFromCommitish(toOrg string, toName string, fromGitURL strin
 		return nil, errors.Wrapf(err, "failed to create GitHub repo %s/%s", toOrg, toName)
 	}
 	dir, err := ioutil.TempDir("", "")
+	log.Logger().Debugf("Using %s to duplicate git repo %s", dir, fromGitURL)
+	// TODO issue-5772 cleanup
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
@@ -855,16 +882,16 @@ func DuplicateGitRepoFromCommitish(toOrg string, toName string, fromGitURL strin
 		return nil, err
 	}
 
-	userDetails := provider.UserAuth()
-	duplicatePushURL, err := gitter.CreateAuthenticatedURL(duplicateInfo.CloneURL, &userDetails)
+	username := provider.CurrentUsername()
+	cloneURLWithUser, err := addUserToURL(duplicateInfo.CloneURL, username)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to create push URL for %s", duplicateInfo.CloneURL)
+		return nil, errors.Wrapf(err, "unable to add username to git url", duplicateInfo.CloneURL)
 	}
-	err = gitter.SetRemoteURL(dir, "origin", duplicateInfo.CloneURL)
+	err = gitter.SetRemoteURL(dir, "origin", cloneURLWithUser)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to set remote url to %s", duplicateInfo.CloneURL)
 	}
-	err = gitter.Push(dir, duplicatePushURL, true, fmt.Sprintf("%s:%s", "HEAD", toBranch))
+	err = gitter.Push(dir, "origin", true, fmt.Sprintf("%s:%s", "HEAD", toBranch))
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to push HEAD to %s", toBranch)
 	}

pkg/versionstream/versionstreamrepo/gitrepo.go

@@ -62,6 +62,19 @@ func cloneJXVersionsRepo(versionRepository string, versionRef string, settings *
 
 	// If the repo already exists let's try to fetch the latest version
 	if exists, err := util.DirExists(wrkDir); err == nil && exists {
+		isBranch, err := gits.RefIsBranch(wrkDir, versionRef, gitter)
+		if err != nil {
+			return "", "", err
+		}
+
+		tag, _, err := gitter.Describe(wrkDir, true, "HEAD", "0", true)
+		if err != nil {
+			return "", "", err
+		}
+		if isBranch && tag == versionRef {
+			return wrkDir, versionRef, nil
+		}
+
 		pullLatest := false
 		if batchMode {
 			pullLatest = true
@@ -96,11 +109,6 @@ func cloneJXVersionsRepo(versionRepository string, versionRef string, settings *
 				return dir, versionRef, nil
 			}
 
-			isBranch, err := gits.RefIsBranch(wrkDir, versionRef, gitter)
-			if err != nil {
-				return "", "", err
-			}
-
 			if versionRef == config.DefaultVersionsRef || isBranch {
 				err = gitter.Checkout(wrkDir, versionRef)
 				if err != nil {