diff --git a/cmd/argocd-application-controller/main.go b/cmd/argocd-application-controller/main.go index 8cf85393e706e..cdf1c6e425bfd 100644 --- a/cmd/argocd-application-controller/main.go +++ b/cmd/argocd-application-controller/main.go @@ -37,6 +37,7 @@ func newCommand() *cobra.Command { clientConfig clientcmd.ClientConfig appResyncPeriod int64 repoServerAddress string + workers int ) var command = cobra.Command{ Use: cliName, @@ -77,7 +78,7 @@ func newCommand() *cobra.Command { defer cancel() log.Infof("Application Controller (version: %s) starting (namespace: %s)", argocd.GetVersion(), namespace) - go appController.Run(ctx, 1) + go appController.Run(ctx, workers) // Wait forever select {} }, @@ -86,6 +87,7 @@ func newCommand() *cobra.Command { clientConfig = cli.AddKubectlFlagsToCmd(&command) command.Flags().Int64Var(&appResyncPeriod, "app-resync", defaultAppResyncPeriod, "Time period in seconds for application resync.") command.Flags().StringVar(&repoServerAddress, "repo-server", "localhost:8081", "Repo server address.") + command.Flags().IntVar(&workers, "workers", 1, "Number of application workers") return &command } diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go index 558643b5d368d..fccbfd42c6585 100644 --- a/reposerver/repository/repository.go +++ b/reposerver/repository/repository.go @@ -37,15 +37,9 @@ func NewService(namespace string, kubeClient kubernetes.Interface, gitClient git } func (s *Service) GetKsonnetApp(ctx context.Context, in *KsonnetAppRequest) (*KsonnetAppResponse, error) { - appRepoPath := path.Join(os.TempDir(), strings.Replace(in.Repo.Repo, "/", "_", -1)) + appRepoPath := tempRepoPath(in.Repo.Repo) s.repoLock.Lock(appRepoPath) - defer func() { - err := s.gitClient.Reset(appRepoPath) - if err != nil { - log.Warn(err) - } - s.repoLock.Unlock(appRepoPath) - }() + defer s.unlockAndResetRepoPath(appRepoPath) ksApp, err := s.getAppSpec(*in.Repo, appRepoPath, in.Revision, in.Path) if err != nil { return nil, err @@ -76,15 +70,9 @@ func ksAppToResponse(ksApp ksutil.KsonnetApp) (*KsonnetAppResponse, error) { } func (s *Service) GenerateManifest(c context.Context, q *ManifestRequest) (*ManifestResponse, error) { - appRepoPath := path.Join(os.TempDir(), strings.Replace(q.Repo.Repo, "/", "_", -1)) + appRepoPath := tempRepoPath(q.Repo.Repo) s.repoLock.Lock(appRepoPath) - defer func() { - err := s.gitClient.Reset(appRepoPath) - if err != nil { - log.Warn(err) - } - s.repoLock.Unlock(appRepoPath) - }() + defer s.unlockAndResetRepoPath(appRepoPath) err := s.gitClient.CloneOrFetch(q.Repo.Repo, q.Repo.Username, q.Repo.Password, q.Repo.SSHPrivateKey, appRepoPath) if err != nil { @@ -183,7 +171,7 @@ func (s *Service) setAppLabels(target *unstructured.Unstructured, appName string // GetEnvParams retrieves Ksonnet environment params in specified repo name and revision func (s *Service) GetEnvParams(c context.Context, q *EnvParamsRequest) (*EnvParamsResponse, error) { - appRepoPath := path.Join(os.TempDir(), strings.Replace(q.Repo.Repo, "/", "_", -1)) + appRepoPath := tempRepoPath(q.Repo.Repo) s.repoLock.Lock(appRepoPath) defer s.repoLock.Unlock(appRepoPath) @@ -211,3 +199,18 @@ func (s *Service) GetEnvParams(c context.Context, q *EnvParamsRequest) (*EnvPara Params: target, }, nil } + +// tempRepoPath returns a formulated temporary directory location to clone a repository +func tempRepoPath(repo string) string { + return path.Join(os.TempDir(), strings.Replace(repo, "/", "_", -1)) +} + +// unlockAndResetRepoPath will reset any local changes in a local git repo and unlock the path +// so that other workers can use the local repo +func (s *Service) unlockAndResetRepoPath(appRepoPath string) { + err := s.gitClient.Reset(appRepoPath) + if err != nil { + log.Warn(err) + } + s.repoLock.Unlock(appRepoPath) +} diff --git a/util/git/client.go b/util/git/client.go index 565d60c853582..8c5889989308f 100644 --- a/util/git/client.go +++ b/util/git/client.go @@ -3,8 +3,10 @@ package git import ( "fmt" "io/ioutil" + "net/url" "os" "os/exec" + "path" "strings" @@ -20,12 +22,61 @@ type Client interface { } // NativeGitClient implements Client interface using git CLI -type NativeGitClient struct { - rootDirectoryPath string +type NativeGitClient struct{} + +// Init initializes a local git repository and sets the remote origin +func (m *NativeGitClient) Init(repo string, repoPath string) error { + log.Infof("Initializing %s to %s", repo, repoPath) + err := os.MkdirAll(repoPath, 0755) + if err != nil { + return err + } + if _, err := runCmd(repoPath, "git", "init"); err != nil { + return err + } + if _, err := runCmd(repoPath, "git", "remote", "add", "origin", repo); err != nil { + return err + } + return nil +} + +// SetCredentials sets a local credentials file to connect to a remote git repository +func (m *NativeGitClient) SetCredentials(repo string, username string, password string, sshPrivateKey string, repoPath string) error { + if password != "" { + gitCredentialsFile := path.Join(repoPath, ".git", "credentials") + repoURL, err := url.ParseRequestURI(repo) + if err != nil { + return err + } + repoURL.User = url.UserPassword(username, password) + cmdURL := repoURL.String() + err = ioutil.WriteFile(gitCredentialsFile, []byte(cmdURL), 0600) + if err != nil { + return fmt.Errorf("failed to set git credentials: %v", err) + } + _, err = runCmd(repoPath, "git", "config", "--local", "credential.helper", fmt.Sprintf("store --file=%s", gitCredentialsFile)) + if err != nil { + return err + } + } + if sshPrivateKey != "" { + sshPrivateKeyFile := path.Join(repoPath, ".git", "ssh-private-key") + err := ioutil.WriteFile(sshPrivateKeyFile, []byte(sshPrivateKey), 0600) + if err != nil { + return fmt.Errorf("failed to set git credentials: %v", err) + } + sshCmd := fmt.Sprintf("ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s", sshPrivateKeyFile) + _, err = runCmd(repoPath, "git", "config", "--local", "core.sshCommand", sshCmd) + if err != nil { + return err + } + } + return nil } // CloneOrFetch either clone or fetch repository into specified directory path. func (m *NativeGitClient) CloneOrFetch(repo string, username string, password string, sshPrivateKey string, repoPath string) error { + log.Debugf("Cloning/Fetching repo %s at %s", repo, repoPath) var needClone bool if _, err := os.Stat(repoPath); os.IsNotExist(err) { needClone = true @@ -35,91 +86,104 @@ func (m *NativeGitClient) CloneOrFetch(repo string, username string, password st _, err = cmd.Output() needClone = err != nil } - - repoURL, env, err := GetGitCommandEnvAndURL(repo, username, password, sshPrivateKey) - if err != nil { - return err - } - if needClone { _, err := exec.Command("rm", "-rf", repoPath).Output() if err != nil { return fmt.Errorf("unable to clean repo cache at %s: %v", repoPath, err) } - - log.Infof("Cloning %s to %s", repo, repoPath) - cmd := exec.Command("git", "clone", repoURL, repoPath) - cmd.Env = env - _, err = cmd.Output() + err = m.Init(repo, repoPath) if err != nil { return fmt.Errorf("unable to clone repository %s: %v", repo, err) } - } else { - log.Infof("Fetching %s", repo) - // Fetch remote changes and delete all local branches - cmd := exec.Command("sh", "-c", "git fetch --all && git checkout --detach HEAD") - cmd.Env = env - cmd.Dir = repoPath - _, err := cmd.Output() - if err != nil { - return fmt.Errorf("unable to fetch repo %s: %v", repoPath, err) - } + } - cmd = exec.Command("sh", "-c", "for i in $(git branch --merged | grep -v \\*); do git branch -D $i; done") - cmd.Dir = repoPath - _, err = cmd.Output() - if err != nil { - return fmt.Errorf("unable to delete local branches for %s: %v", repoPath, err) + err := m.SetCredentials(repo, username, password, sshPrivateKey, repoPath) + if err != nil { + return err + } + // Fetch remote changes + if _, err = runCmd(repoPath, "git", "fetch", "origin"); err != nil { + return err + } + // git fetch does not update the HEAD reference. The following command will update the local + // knowledge of what remote considers the “default branch” + // See: https://stackoverflow.com/questions/8839958/how-does-origin-head-get-set + if _, err := runCmd(repoPath, "git", "remote", "set-head", "origin", "-a"); err != nil { + return err + } + // Delete all local branches (we must first detach so we are not checked out a branch we are about to delete) + if _, err = runCmd(repoPath, "git", "checkout", "--detach", "origin/HEAD"); err != nil { + return err + } + branchesOut, err := runCmd(repoPath, "git", "for-each-ref", "--format=%(refname:short)", "refs/heads/") + if err != nil { + return err + } + branchesOut = strings.TrimSpace(branchesOut) + if branchesOut != "" { + branches := strings.Split(branchesOut, "\n") + args := []string{"branch", "-D"} + args = append(args, branches...) + if _, err = runCmd(repoPath, "git", args...); err != nil { + return err } - } return nil } -// Reset resets local changes +// Reset resets local changes in a repository func (m *NativeGitClient) Reset(repoPath string) error { - cmd := exec.Command("sh", "-c", "git reset --hard HEAD && git clean -f") - cmd.Dir = repoPath - _, err := cmd.Output() - if err != nil { - return fmt.Errorf("unable to reset repository %s: %v", repoPath, err) + if _, err := runCmd(repoPath, "git", "reset", "--hard", "origin/HEAD"); err != nil { + return err + } + if _, err := runCmd(repoPath, "git", "clean", "-f"); err != nil { + return err } - return nil } // Checkout checkout specified git sha -func (m *NativeGitClient) Checkout(repoPath string, sha string) (string, error) { - if sha == "" { - sha = "origin/HEAD" +func (m *NativeGitClient) Checkout(repoPath string, revision string) (string, error) { + if revision == "" || revision == "HEAD" { + revision = "origin/HEAD" } - checkoutCmd := exec.Command("git", "checkout", sha) - checkoutCmd.Dir = repoPath - _, err := checkoutCmd.Output() - if err != nil { - return "", fmt.Errorf("unable to checkout revision %s: %v", sha, err) + if _, err := runCmd(repoPath, "git", "checkout", revision); err != nil { + return "", err } return m.CommitSHA(repoPath) } // CommitSHA returns current commit sha from `git rev-parse HEAD` func (m *NativeGitClient) CommitSHA(repoPath string) (string, error) { - revisionCmd := exec.Command("git", "rev-parse", "HEAD") - revisionCmd.Dir = repoPath - output, err := revisionCmd.Output() + out, err := runCmd(repoPath, "git", "rev-parse", "HEAD") if err != nil { return "", err } - return strings.TrimSpace(string(output)), nil + return strings.TrimSpace(out), nil } // NewNativeGitClient creates new instance of NativeGitClient func NewNativeGitClient() (Client, error) { - rootDirPath, err := ioutil.TempDir("", "argo-git") + return &NativeGitClient{}, nil +} + +// runCmd is a convenience function to run a command in a given directory and return its output +func runCmd(cwd string, command string, args ...string) (string, error) { + cmd := exec.Command(command, args...) + log.Debug(strings.Join(cmd.Args, " ")) + cmd.Dir = cwd + out, err := cmd.Output() + if len(out) > 0 { + log.Debug(string(out)) + } if err != nil { - return nil, err + exErr, ok := err.(*exec.ExitError) + if ok { + errOutput := strings.Split(string(exErr.Stderr), "\n")[0] + log.Debug(errOutput) + return string(out), fmt.Errorf("'%s' failed: %v", strings.Join(cmd.Args, " "), errOutput) + } + return string(out), fmt.Errorf("'%s' failed: %v", strings.Join(cmd.Args, " "), err) } - return &NativeGitClient{ - rootDirectoryPath: rootDirPath, - }, nil + return string(out), nil }