From e24a612b26a72ebf51f4fc9e627408535b6fa4e1 Mon Sep 17 00:00:00 2001 From: Hardy Ferentschik Date: Fri, 10 Jan 2020 20:45:29 +0100 Subject: [PATCH] feat: Handling git credentials via credential helper fixes #5772 --- pkg/cmd/cmd.go | 8 ++- .../git/credentials/step_git_credentials.go | 51 +++++++++++++- .../step/verify/step_verify_environments.go | 8 +-- pkg/gits/git_cli.go | 36 ++++++++++ pkg/gits/helpers.go | 67 +++++++++++++------ .../versionstreamrepo/gitrepo.go | 18 +++-- 6 files changed, 154 insertions(+), 34 deletions(-) diff --git a/pkg/cmd/cmd.go b/pkg/cmd/cmd.go index fb0a0ab3a7..e71121ebd8 100644 --- a/pkg/cmd/cmd.go +++ b/pkg/cmd/cmd.go @@ -79,7 +79,7 @@ func NewJXCommand(f clients.Factory, in terminal.FileReader, out terminal.FileWr rootCommand := &cobra.Command{ Use: "jx", Short: "jx is a command line tool for working with Jenkins X", - PersistentPreRun: setLoggingLevel, + PersistentPreRun: persistentPreRun, Run: runHelp, } @@ -304,7 +304,11 @@ func fullPath(command *cobra.Command) string { return name } -func setLoggingLevel(cmd *cobra.Command, args []string) { +func persistentPreRun(cmd *cobra.Command, args []string) { + setLoggingLevel(cmd) +} + +func setLoggingLevel(cmd *cobra.Command) { verbose, err := strconv.ParseBool(cmd.Flag(opts.OptionVerbose).Value.String()) if err != nil { log.Logger().Errorf("Unable to check if the verbose flag is set") diff --git a/pkg/cmd/step/git/credentials/step_git_credentials.go b/pkg/cmd/step/git/credentials/step_git_credentials.go index 8727c9fbf1..b5694d8981 100644 --- a/pkg/cmd/step/git/credentials/step_git_credentials.go +++ b/pkg/cmd/step/git/credentials/step_git_credentials.go @@ -7,6 +7,7 @@ import ( "net/url" "os" "path/filepath" + "regexp" "github.com/jenkins-x/jx/pkg/auth" "github.com/jenkins-x/jx/pkg/cmd/opts/step" @@ -37,6 +38,7 @@ type StepGitCredentialsOptions struct { GitHubAppOwner string GitKind string CredentialsSecret string + AskPass bool } type credentials struct { @@ -57,6 +59,9 @@ var ( # generate the Git credentials to a output file jx step git credentials -o /tmp/mycreds + + # respond to a GIT_ASKPASS request + jx step git credentials --ask-pass `) ) @@ -82,6 +87,7 @@ func NewCmdStepGitCredentials(commonOpts *opts.CommonOptions) *cobra.Command { cmd.Flags().StringVarP(&options.GitHubAppOwner, optionGitHubAppOwner, "g", "", "The owner (organisation or user name) if using GitHub App based tokens") cmd.Flags().StringVarP(&options.CredentialsSecret, "credentials-secret", "s", "", "The secret name to read the credentials from") cmd.Flags().StringVarP(&options.GitKind, "git-kind", "", "", "The git kind. e.g. github, bitbucketserver etc") + cmd.Flags().BoolVar(&options.AskPass, "ask-pass", false, "respond to a GIT_ASKPASS request") return cmd } @@ -142,9 +148,22 @@ func (o *StepGitCredentialsOptions) Run() error { credentials, err := o.CreateGitCredentialsFromAuthService(authConfigSvc) if err != nil { + return errors.Wrap(err, "creating git credentials") } - return o.createGitCredentialsFile(outFile, credentials) + + if o.AskPass { + response := o.askPass(os.Args[len(os.Args)-1], credentials) + fmt.Println(response) + return nil + } else { + outFile, err := o.determineOutputFile() + if err != nil { + return err + } + + return o.createGitCredentialsFile(outFile, credentials) + } } func (o *StepGitCredentialsOptions) GitCredentialsFileData(credentials []credentials) ([]byte, error) { @@ -168,6 +187,36 @@ func (o *StepGitCredentialsOptions) GitCredentialsFileData(credentials []credent return buffer.Bytes(), nil } +// TODO issue-5772 handle input from GIT_ASKPASS properly by parsing the git provider +func (o *StepGitCredentialsOptions) askPass(question string, credentials []credentials) string { + re, err := regexp.Compile(`^.*'(?Phttp.*)':.*$`) + if err != nil { + log.Logger().Errorf("unable to compile regexp: %s", err.Error()) + return "" + } + + matches := re.FindStringSubmatch(question) + + if matches == nil { + log.Logger().Errorf("unexpected password challenge format: %s", question) + return "" + } + + gitURL, err := url.Parse(matches[1]) + if err != nil { + log.Logger().Errorf("invalid git URL: %s", err.Error()) + return "" + } + + for _, creds := range credentials { + if gitURL.User.Username() == creds.user { + return creds.password + } + } + + return "" +} + func (o *StepGitCredentialsOptions) determineOutputFile() (string, error) { outFile := o.OutputFile if outFile == "" { diff --git a/pkg/cmd/step/verify/step_verify_environments.go b/pkg/cmd/step/verify/step_verify_environments.go index 58837c3b20..4235767554 100644 --- a/pkg/cmd/step/verify/step_verify_environments.go +++ b/pkg/cmd/step/verify/step_verify_environments.go @@ -487,12 +487,8 @@ func (o *StepVerifyEnvironmentsOptions) pushDevEnvironmentUpdates(environmentRep } } - userDetails := provider.UserAuth() - authenticatedPushURL, err := gitter.CreateAuthenticatedURL(environmentRepo.CloneURL, &userDetails) - if err != nil { - return errors.Wrapf(err, "failed to create push URL for %s", environmentRepo.CloneURL) - } - err = gitter.Push(localRepoDir, authenticatedPushURL, true, "master") + // TODO issue-5772 determine final remote + err = gitter.Push(localRepoDir, "origin", true, "master") if err != nil { return errors.Wrapf(err, "unable to push %s to %s", localRepoDir, environmentRepo.URL) } diff --git a/pkg/gits/git_cli.go b/pkg/gits/git_cli.go index 9f6f8d5898..b790481f32 100644 --- a/pkg/gits/git_cli.go +++ b/pkg/gits/git_cli.go @@ -24,6 +24,11 @@ import ( const ( replaceInvalidBranchChars = '_' + askPassBin = "git-askpass" + askPassNonWindows = `#!/usr/bin/env sh + +jx step git credentials --ask-pass "$@" +` // #nosec ) var ( @@ -43,9 +48,40 @@ func NewGitCLI() *GitCLI { } // Ensure that error output is in English so parsing work cli.Env["LC_ALL"] = "C" + cli.Env["JX_LOG_LEVEL"] = "error" + cli.configureGitAskPass() return cli } +func (g *GitCLI) configureGitAskPass() { + // TODO issue-5772 add a batch file for windows + configDir, err := util.ConfigDir() + if err != nil { + log.Logger().Errorf("unable to determine current JX_HOME: %s", err.Error()) + } + + binPath := filepath.Join(configDir, "bin") + err = os.MkdirAll(binPath, os.ModePerm) + if err != nil { + log.Logger().Errorf("unable to create bin directory: %s", err.Error()) + } + askPassBin := filepath.Join(binPath, askPassBin) + exists, err := util.FileExists(askPassBin) + if err != nil { + log.Logger().Errorf("unable to determine file stats for %s: %s", askPassBin, err.Error()) + } + + if !exists { + data := []byte(askPassNonWindows) + err := ioutil.WriteFile(askPassBin, data, 0750) + if err != nil { + log.Logger().Errorf("unable to write %s: %s", askPassBin, err.Error()) + } + } + + g.Env["GIT_ASKPASS"] = askPassBin +} + // FindGitConfigDir tries to find the `.git` directory either in the current directory or in parent directories func (g *GitCLI) FindGitConfigDir(dir string) (string, string, error) { d := dir diff --git a/pkg/gits/helpers.go b/pkg/gits/helpers.go index 1135a9c2f1..9aa7da5043 100644 --- a/pkg/gits/helpers.go +++ b/pkg/gits/helpers.go @@ -11,6 +11,7 @@ import ( "reflect" "sort" "strings" + "time" "gopkg.in/src-d/go-git.v4/config" @@ -432,6 +433,8 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin return "", "", nil, nil, errors.Wrapf(err, "failed to parse gitter URL %s", gitURL) } + log.Logger().Debugf("ForkAndPullRepo gitURL: %s dir: %s baseRef: %s branchName: %s forkName: %s", gitURL, dir, baseRef, branchName, forkName) + username := provider.CurrentUsername() originalOrg := originalInfo.Organisation originalRepo := originalInfo.Name @@ -517,6 +520,7 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin return "", "", nil, nil, errors.WithStack(err) } } else { + // TODO issue-5572 this will not stash everything err = gitter.StashPush(dir) stashed = true if err != nil { @@ -526,23 +530,26 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin // The long form of "git clone" has the advantage of working fine on an existing git repo and avoids checking out master // and then another branch - err = gitter.SetRemoteURL(dir, originRemote, originURL) + originURLWithUser, err := addUserToURL(originURL, username) + if err != nil { + return "", "", nil, nil, errors.Wrapf(err, "unable to add username to git url", originURL) + } + err = gitter.SetRemoteURL(dir, originRemote, originURLWithUser) if err != nil { return "", "", nil, nil, errors.Wrapf(err, "failed to set %s url to %s", originRemote, originURL) } if fork { upstreamRemote = "upstream" - err := gitter.SetRemoteURL(dir, upstreamRemote, upstreamInfo.CloneURL) + upstreamURLWithUser, err := addUserToURL(upstreamInfo.CloneURL, username) + if err != nil { + return "", "", nil, nil, errors.Wrapf(err, "unable to add username to git url", upstreamInfo.CloneURL) + } + err = gitter.SetRemoteURL(dir, upstreamRemote, upstreamURLWithUser) if err != nil { return "", "", nil, nil, errors.Wrapf(err, "setting remote upstream %q in forked environment repo", gitURL) } } - userDetails := provider.UserAuth() - originFetchURL, err := gitter.CreateAuthenticatedURL(originURL, &userDetails) - if err != nil { - return "", "", nil, nil, errors.Wrapf(err, "failed to create authenticated fetch URL for %s", originURL) - } branchNameUUID, err := uuid.NewV4() if err != nil { return "", "", nil, nil, errors.WithStack(err) @@ -550,7 +557,7 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin originFetchBranch := branchNameUUID.String() var upstreamFetchBranch string - err = gitter.FetchBranch(dir, originFetchURL, fmt.Sprintf("%s:%s", branchName, originFetchBranch)) + err = gitter.FetchBranch(dir, "origin", fmt.Sprintf("%s:%s", branchName, originFetchBranch)) if err != nil { if IsCouldntFindRemoteRefError(err, branchName) { // We can safely ignore missing remote branches, as they just don't exist @@ -561,21 +568,17 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin } if upstreamRemote != originRemote || baseRef != branchName { - upstreamFetchURL, err := gitter.CreateAuthenticatedURL(upstreamInfo.CloneURL, &userDetails) branchNameUUID, err := uuid.NewV4() if err != nil { return "", "", nil, nil, errors.WithStack(err) } upstreamFetchBranch = branchNameUUID.String() - if err != nil { - return "", "", nil, nil, errors.Wrapf(err, "failed to create authenticated fetch URL for %s", upstreamInfo.CloneURL) - } + // We're going to start our work from baseRef on the upstream - err = gitter.FetchBranch(dir, upstreamFetchURL, fmt.Sprintf("%s:%s", baseRef, upstreamFetchBranch)) + err = gitter.FetchBranch(dir, upstreamRemote, fmt.Sprintf("%s:%s", baseRef, upstreamFetchBranch)) if err != nil { return "", "", nil, nil, errors.WithStack(err) } - } // Work out what branch to use and check it out @@ -599,6 +602,10 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin return "", "", nil, nil, errors.WithStack(err) } } else if dirExists { + // TODO issue-5772 tmp + log.Logger().Info("Sleep 5 minutes") + time.Sleep(5 * time.Minute) + log.Logger().Info("Resuming") toCherryPick, err = gitter.GetCommitsNotOnAnyRemote(dir, localBranchName) if err != nil { return "", "", nil, nil, errors.WithStack(err) @@ -616,7 +623,11 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin // Merge in any local committed changes we found if len(toCherryPick) > 0 { - log.Logger().Infof("Attempting to cherry pick commits that were on %s but not yet pushed", localBranchName) + var shas = make([]string, len(toCherryPick)) + for _, commit := range toCherryPick { + shas = append(shas, commit.SHA) + } + log.Logger().Debugf("Attempting to cherry pick commits %s that were on %s but not yet pushed", shas, localBranchName) } for _, c := range toCherryPick { err = gitter.CherryPick(dir, c.SHA) @@ -658,6 +669,20 @@ func ForkAndPullRepo(gitURL string, dir string, baseRef string, branchName strin return dir, baseRef, upstreamInfo, forkInfo, nil } +func addUserToURL(gitURL string, user string) (string, error) { + u, err := url.Parse(gitURL) + if err != nil { + return "", errors.Wrapf(err, "invalid git URL: %s", gitURL) + } + + if user != "" { + userInfo := url.User(user) + u.User = userInfo + } + + return u.String(), nil +} + // A PullRequestFilter defines a filter for finding pull requests type PullRequestFilter struct { Labels []string @@ -805,6 +830,8 @@ func DuplicateGitRepoFromCommitish(toOrg string, toName string, fromGitURL strin return nil, errors.Wrapf(err, "failed to create GitHub repo %s/%s", toOrg, toName) } dir, err := ioutil.TempDir("", "") + log.Logger().Debugf("Using %s to duplicate git repo %s", dir, fromGitURL) + // TODO issue-5772 cleanup if err != nil { return nil, errors.WithStack(err) } @@ -855,16 +882,16 @@ func DuplicateGitRepoFromCommitish(toOrg string, toName string, fromGitURL strin return nil, err } - userDetails := provider.UserAuth() - duplicatePushURL, err := gitter.CreateAuthenticatedURL(duplicateInfo.CloneURL, &userDetails) + username := provider.CurrentUsername() + cloneURLWithUser, err := addUserToURL(duplicateInfo.CloneURL, username) if err != nil { - return nil, errors.Wrapf(err, "failed to create push URL for %s", duplicateInfo.CloneURL) + return nil, errors.Wrapf(err, "unable to add username to git url", duplicateInfo.CloneURL) } - err = gitter.SetRemoteURL(dir, "origin", duplicateInfo.CloneURL) + err = gitter.SetRemoteURL(dir, "origin", cloneURLWithUser) if err != nil { return nil, errors.Wrapf(err, "failed to set remote url to %s", duplicateInfo.CloneURL) } - err = gitter.Push(dir, duplicatePushURL, true, fmt.Sprintf("%s:%s", "HEAD", toBranch)) + err = gitter.Push(dir, "origin", true, fmt.Sprintf("%s:%s", "HEAD", toBranch)) if err != nil { return nil, errors.Wrapf(err, "failed to push HEAD to %s", toBranch) } diff --git a/pkg/versionstream/versionstreamrepo/gitrepo.go b/pkg/versionstream/versionstreamrepo/gitrepo.go index 565cb7cadd..d7c4b889c2 100644 --- a/pkg/versionstream/versionstreamrepo/gitrepo.go +++ b/pkg/versionstream/versionstreamrepo/gitrepo.go @@ -62,6 +62,19 @@ func cloneJXVersionsRepo(versionRepository string, versionRef string, settings * // If the repo already exists let's try to fetch the latest version if exists, err := util.DirExists(wrkDir); err == nil && exists { + isBranch, err := gits.RefIsBranch(wrkDir, versionRef, gitter) + if err != nil { + return "", "", err + } + + tag, _, err := gitter.Describe(wrkDir, true, "HEAD", "0", true) + if err != nil { + return "", "", err + } + if isBranch && tag == versionRef { + return wrkDir, versionRef, nil + } + pullLatest := false if batchMode { pullLatest = true @@ -96,11 +109,6 @@ func cloneJXVersionsRepo(versionRepository string, versionRef string, settings * return dir, versionRef, nil } - isBranch, err := gits.RefIsBranch(wrkDir, versionRef, gitter) - if err != nil { - return "", "", err - } - if versionRef == config.DefaultVersionsRef || isBranch { err = gitter.Checkout(wrkDir, versionRef) if err != nil {