Refactor git library

cmd/argocd-application-controller/main.go

@@ -37,6 +37,7 @@ func newCommand() *cobra.Command {
 		clientConfig      clientcmd.ClientConfig
 		appResyncPeriod   int64
 		repoServerAddress string
+		workers           int
 	)
 	var command = cobra.Command{
 		Use:   cliName,
@@ -77,7 +78,7 @@ func newCommand() *cobra.Command {
 			defer cancel()
 
 			log.Infof("Application Controller (version: %s) starting (namespace: %s)", argocd.GetVersion(), namespace)
-			go appController.Run(ctx, 1)
+			go appController.Run(ctx, workers)
 			// Wait forever
 			select {}
 		},
@@ -86,6 +87,7 @@ func newCommand() *cobra.Command {
 	clientConfig = cli.AddKubectlFlagsToCmd(&command)
 	command.Flags().Int64Var(&appResyncPeriod, "app-resync", defaultAppResyncPeriod, "Time period in seconds for application resync.")
 	command.Flags().StringVar(&repoServerAddress, "repo-server", "localhost:8081", "Repo server address.")
+	command.Flags().IntVar(&workers, "workers", 1, "Number of application workers")
 	return &command
 }
 

reposerver/repository/repository.go

@@ -37,15 +37,9 @@ func NewService(namespace string, kubeClient kubernetes.Interface, gitClient git
 }
 
 func (s *Service) GetKsonnetApp(ctx context.Context, in *KsonnetAppRequest) (*KsonnetAppResponse, error) {
-	appRepoPath := path.Join(os.TempDir(), strings.Replace(in.Repo.Repo, "/", "_", -1))
+	appRepoPath := tempRepoPath(in.Repo.Repo)
 	s.repoLock.Lock(appRepoPath)
-	defer func() {
-		err := s.gitClient.Reset(appRepoPath)
-		if err != nil {
-			log.Warn(err)
-		}
-		s.repoLock.Unlock(appRepoPath)
-	}()
+	defer s.unlockAndResetRepoPath(appRepoPath)
 	ksApp, err := s.getAppSpec(*in.Repo, appRepoPath, in.Revision, in.Path)
 	if err != nil {
 		return nil, err
@@ -76,15 +70,9 @@ func ksAppToResponse(ksApp ksutil.KsonnetApp) (*KsonnetAppResponse, error) {
 }
 
 func (s *Service) GenerateManifest(c context.Context, q *ManifestRequest) (*ManifestResponse, error) {
-	appRepoPath := path.Join(os.TempDir(), strings.Replace(q.Repo.Repo, "/", "_", -1))
+	appRepoPath := tempRepoPath(q.Repo.Repo)
 	s.repoLock.Lock(appRepoPath)
-	defer func() {
-		err := s.gitClient.Reset(appRepoPath)
-		if err != nil {
-			log.Warn(err)
-		}
-		s.repoLock.Unlock(appRepoPath)
-	}()
+	defer s.unlockAndResetRepoPath(appRepoPath)
 
 	err := s.gitClient.CloneOrFetch(q.Repo.Repo, q.Repo.Username, q.Repo.Password, q.Repo.SSHPrivateKey, appRepoPath)
 	if err != nil {
@@ -183,7 +171,7 @@ func (s *Service) setAppLabels(target *unstructured.Unstructured, appName string
 
 // GetEnvParams retrieves Ksonnet environment params in specified repo name and revision
 func (s *Service) GetEnvParams(c context.Context, q *EnvParamsRequest) (*EnvParamsResponse, error) {
-	appRepoPath := path.Join(os.TempDir(), strings.Replace(q.Repo.Repo, "/", "_", -1))
+	appRepoPath := tempRepoPath(q.Repo.Repo)
 	s.repoLock.Lock(appRepoPath)
 	defer s.repoLock.Unlock(appRepoPath)
 
@@ -211,3 +199,18 @@ func (s *Service) GetEnvParams(c context.Context, q *EnvParamsRequest) (*EnvPara
 		Params: target,
 	}, nil
 }
+
+// tempRepoPath returns a formulated temporary directory location to clone a repository
+func tempRepoPath(repo string) string {
+	return path.Join(os.TempDir(), strings.Replace(repo, "/", "_", -1))
+}
+
+// unlockAndResetRepoPath will reset any local changes in a local git repo and unlock the path
+// so that other workers can use the local repo
+func (s *Service) unlockAndResetRepoPath(appRepoPath string) {
+	err := s.gitClient.Reset(appRepoPath)
+	if err != nil {
+		log.Warn(err)
+	}
+	s.repoLock.Unlock(appRepoPath)
+}

util/git/client.go

@@ -3,8 +3,10 @@ package git
 import (
 	"fmt"
 	"io/ioutil"
+	"net/url"
 	"os"
 	"os/exec"
+	"path"
 
 	"strings"
 
@@ -20,12 +22,61 @@ type Client interface {
 }
 
 // NativeGitClient implements Client interface using git CLI
-type NativeGitClient struct {
-	rootDirectoryPath string
+type NativeGitClient struct{}
+
+// Init initializes a local git repository and sets the remote origin
+func (m *NativeGitClient) Init(repo string, repoPath string) error {
+	log.Infof("Initializing %s to %s", repo, repoPath)
+	err := os.MkdirAll(repoPath, 0755)
+	if err != nil {
+		return err
+	}
+	if _, err := runCmd(repoPath, "git", "init"); err != nil {
+		return err
+	}
+	if _, err := runCmd(repoPath, "git", "remote", "add", "origin", repo); err != nil {
+		return err
+	}
+	return nil
+}
+
+// SetCredentials sets a local credentials file to connect to a remote git repository
+func (m *NativeGitClient) SetCredentials(repo string, username string, password string, sshPrivateKey string, repoPath string) error {
+	if password != "" {
+		gitCredentialsFile := path.Join(repoPath, ".git", "credentials")
+		repoURL, err := url.ParseRequestURI(repo)
+		if err != nil {
+			return err
+		}
+		repoURL.User = url.UserPassword(username, password)
+		cmdURL := repoURL.String()
+		err = ioutil.WriteFile(gitCredentialsFile, []byte(cmdURL), 0600)
+		if err != nil {
+			return fmt.Errorf("failed to set git credentials: %v", err)
+		}
+		_, err = runCmd(repoPath, "git", "config", "--local", "credential.helper", fmt.Sprintf("store --file=%s", gitCredentialsFile))
+		if err != nil {
+			return err
+		}
+	}
+	if sshPrivateKey != "" {
+		sshPrivateKeyFile := path.Join(repoPath, ".git", "ssh-private-key")
+		err := ioutil.WriteFile(sshPrivateKeyFile, []byte(sshPrivateKey), 0600)
+		if err != nil {
+			return fmt.Errorf("failed to set git credentials: %v", err)
+		}
+		sshCmd := fmt.Sprintf("ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s", sshPrivateKeyFile)
+		_, err = runCmd(repoPath, "git", "config", "--local", "core.sshCommand", sshCmd)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // CloneOrFetch either clone or fetch repository into specified directory path.
 func (m *NativeGitClient) CloneOrFetch(repo string, username string, password string, sshPrivateKey string, repoPath string) error {
+	log.Debugf("Cloning/Fetching repo %s at %s", repo, repoPath)
 	var needClone bool
 	if _, err := os.Stat(repoPath); os.IsNotExist(err) {
 		needClone = true
@@ -35,91 +86,104 @@ func (m *NativeGitClient) CloneOrFetch(repo string, username string, password st
 		_, err = cmd.Output()
 		needClone = err != nil
 	}
-
-	repoURL, env, err := GetGitCommandEnvAndURL(repo, username, password, sshPrivateKey)
-	if err != nil {
-		return err
-	}
-
 	if needClone {
 		_, err := exec.Command("rm", "-rf", repoPath).Output()
 		if err != nil {
 			return fmt.Errorf("unable to clean repo cache at %s: %v", repoPath, err)
 		}
-
-		log.Infof("Cloning %s to %s", repo, repoPath)
-		cmd := exec.Command("git", "clone", repoURL, repoPath)
-		cmd.Env = env
-		_, err = cmd.Output()
+		err = m.Init(repo, repoPath)
 		if err != nil {
 			return fmt.Errorf("unable to clone repository %s: %v", repo, err)
 		}
-	} else {
-		log.Infof("Fetching %s", repo)
-		// Fetch remote changes and delete all local branches
-		cmd := exec.Command("sh", "-c", "git fetch --all && git checkout --detach HEAD")
-		cmd.Env = env
-		cmd.Dir = repoPath
-		_, err := cmd.Output()
-		if err != nil {
-			return fmt.Errorf("unable to fetch repo %s: %v", repoPath, err)
-		}
+	}
 
-		cmd = exec.Command("sh", "-c", "for i in $(git branch --merged | grep -v \\*); do git branch -D $i; done")
-		cmd.Dir = repoPath
-		_, err = cmd.Output()
-		if err != nil {
-			return fmt.Errorf("unable to delete local branches for %s: %v", repoPath, err)
+	err := m.SetCredentials(repo, username, password, sshPrivateKey, repoPath)
+	if err != nil {
+		return err
+	}
+	// Fetch remote changes
+	if _, err = runCmd(repoPath, "git", "fetch", "origin"); err != nil {
+		return err
+	}
+	// git fetch does not update the HEAD reference. The following command will update the local
+	// knowledge of what remote considers the “default branch”
+	// See: https://stackoverflow.com/questions/8839958/how-does-origin-head-get-set
+	if _, err := runCmd(repoPath, "git", "remote", "set-head", "origin", "-a"); err != nil {
+		return err
+	}
+	// Delete all local branches (we must first detach so we are not checked out a branch we are about to delete)
+	if _, err = runCmd(repoPath, "git", "checkout", "--detach", "origin/HEAD"); err != nil {
+		return err
+	}
+	branchesOut, err := runCmd(repoPath, "git", "for-each-ref", "--format=%(refname:short)", "refs/heads/")
+	if err != nil {
+		return err
+	}
+	branchesOut = strings.TrimSpace(branchesOut)
+	if branchesOut != "" {
+		branches := strings.Split(branchesOut, "\n")
+		args := []string{"branch", "-D"}
+		args = append(args, branches...)
+		if _, err = runCmd(repoPath, "git", args...); err != nil {
+			return err
 		}
-
 	}
 	return nil
 }
 
-// Reset resets local changes
+// Reset resets local changes in a repository
 func (m *NativeGitClient) Reset(repoPath string) error {
-	cmd := exec.Command("sh", "-c", "git reset --hard HEAD && git clean -f")
-	cmd.Dir = repoPath
-	_, err := cmd.Output()
-	if err != nil {
-		return fmt.Errorf("unable to reset repository %s: %v", repoPath, err)
+	if _, err := runCmd(repoPath, "git", "reset", "--hard", "origin/HEAD"); err != nil {
+		return err
+	}
+	if _, err := runCmd(repoPath, "git", "clean", "-f"); err != nil {
+		return err
 	}
-
 	return nil
 }
 
 // Checkout checkout specified git sha
-func (m *NativeGitClient) Checkout(repoPath string, sha string) (string, error) {
-	if sha == "" {
-		sha = "origin/HEAD"
+func (m *NativeGitClient) Checkout(repoPath string, revision string) (string, error) {
+	if revision == "" || revision == "HEAD" {
+		revision = "origin/HEAD"
 	}
-	checkoutCmd := exec.Command("git", "checkout", sha)
-	checkoutCmd.Dir = repoPath
-	_, err := checkoutCmd.Output()
-	if err != nil {
-		return "", fmt.Errorf("unable to checkout revision %s: %v", sha, err)
+	if _, err := runCmd(repoPath, "git", "checkout", revision); err != nil {
+		return "", err
 	}
 	return m.CommitSHA(repoPath)
 }
 
 // CommitSHA returns current commit sha from `git rev-parse HEAD`
 func (m *NativeGitClient) CommitSHA(repoPath string) (string, error) {
-	revisionCmd := exec.Command("git", "rev-parse", "HEAD")
-	revisionCmd.Dir = repoPath
-	output, err := revisionCmd.Output()
+	out, err := runCmd(repoPath, "git", "rev-parse", "HEAD")
 	if err != nil {
 		return "", err
 	}
-	return strings.TrimSpace(string(output)), nil
+	return strings.TrimSpace(out), nil
 }
 
 // NewNativeGitClient creates new instance of NativeGitClient
 func NewNativeGitClient() (Client, error) {
-	rootDirPath, err := ioutil.TempDir("", "argo-git")
+	return &NativeGitClient{}, nil
+}
+
+// runCmd is a convenience function to run a command in a given directory and return its output
+func runCmd(cwd string, command string, args ...string) (string, error) {
+	cmd := exec.Command(command, args...)
+	log.Debug(strings.Join(cmd.Args, " "))
+	cmd.Dir = cwd
+	out, err := cmd.Output()
+	if len(out) > 0 {
+		log.Debug(string(out))
+	}
 	if err != nil {
-		return nil, err
+		exErr, ok := err.(*exec.ExitError)
+		if ok {
+			errOutput := strings.Split(string(exErr.Stderr), "\n")[0]
+			log.Debug(errOutput)
+			return string(out), fmt.Errorf("'%s' failed: %v", strings.Join(cmd.Args, " "), errOutput)
+		}
+		return string(out), fmt.Errorf("'%s' failed: %v", strings.Join(cmd.Args, " "), err)
 	}
-	return &NativeGitClient{
-		rootDirectoryPath: rootDirPath,
-	}, nil
+	return string(out), nil
 }