chore: code-refactoring

App.go

@@ -24,7 +24,7 @@ import (
 	pubsub "github.com/devtron-labs/common-lib/pubsub-lib"
 	"github.com/devtron-labs/git-sensor/api"
 	"github.com/devtron-labs/git-sensor/bean"
-	"github.com/devtron-labs/git-sensor/internal/middleware"
+	"github.com/devtron-labs/git-sensor/internals/middleware"
 	"github.com/devtron-labs/git-sensor/pkg/git"
 	pb "github.com/devtron-labs/protos/gitSensor"
 	"github.com/go-pg/pg"

api/GrpcHandler.go

@@ -2,7 +2,7 @@ package api
 
 import (
 	"context"
-	"github.com/devtron-labs/git-sensor/internal/sql"
+	"github.com/devtron-labs/git-sensor/internals/sql"
 	"github.com/devtron-labs/git-sensor/pkg"
 	"github.com/devtron-labs/git-sensor/pkg/git"
 	pb "github.com/devtron-labs/protos/gitSensor"
@@ -355,6 +355,7 @@ func (impl *GrpcHandlerImpl) GetCommitMetadataForPipelineMaterial(ctx context.Co
 
 		return nil, err
 	}
+
 	if res == nil {
 		res1 := &pb.GitCommit{}
 		return res1, nil

api/GrpcHandler_test.go

@@ -3,7 +3,7 @@ package api
 import (
 	"context"
 	"encoding/json"
-	"github.com/devtron-labs/git-sensor/internal/sql"
+	"github.com/devtron-labs/git-sensor/internals/sql"
 	"github.com/devtron-labs/git-sensor/pkg"
 	"github.com/devtron-labs/git-sensor/pkg/git"
 	"github.com/devtron-labs/git-sensor/pkg/mocks"

api/RestHandler.go

@@ -18,7 +18,7 @@ package api
 
 import (
 	"encoding/json"
-	"github.com/devtron-labs/git-sensor/internal/sql"
+	"github.com/devtron-labs/git-sensor/internals/sql"
 	"github.com/devtron-labs/git-sensor/pkg"
 	"github.com/devtron-labs/git-sensor/pkg/git"
 	"github.com/gorilla/mux"

internal/Configuration.go → internals/Configuration.go

@@ -1,13 +1,12 @@
-package internal
+package internals
 
-import (
-	"github.com/caarlos0/env"
-)
+import "github.com/caarlos0/env"
 
 type Configuration struct {
 	CommitStatsTimeoutInSec int    `env:"COMMIT_STATS_TIMEOUT_IN_SEC" envDefault:"2"`
 	EnableFileStats         bool   `env:"ENABLE_FILE_STATS" envDefault:"false"`
 	GitHistoryCount         int    `env:"GIT_HISTORY_COUNT" envDefault:"15"`
+	CloningMode             string `env:"CLONING_MODE" envDefault:"FULL"`
 	MinLimit                int    `env:"MIN_LIMIT_FOR_PVC" envDefault:"1"` // in MB
 	UseGitCli               bool   `env:"USE_GIT_CLI" envDefault:"false"`
 	AnalyticsDebug          bool   `env:"ANALYTICS_DEBUG" envDefault:"false"`

internal/Lockutil.go → internals/Lockutil.go

@@ -14,7 +14,7 @@
  *  limitations under the License.
  */
 
-package internal
+package internals
 
 import (
 	"go.uber.org/zap"

internal/util/HelperUtil.go → internals/util/HelperUtil.go

@@ -1,7 +1,7 @@
 package util
 
 import (
-	"github.com/devtron-labs/git-sensor/internal/sql"
+	"github.com/devtron-labs/git-sensor/internals/sql"
 	"strings"
 )
 

pkg/RepoManages.go

@@ -20,9 +20,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/devtron-labs/git-sensor/internal"
-	"github.com/devtron-labs/git-sensor/internal/sql"
-	"github.com/devtron-labs/git-sensor/internal/util"
+	"github.com/devtron-labs/git-sensor/internals"
+	"github.com/devtron-labs/git-sensor/internals/sql"
+	"github.com/devtron-labs/git-sensor/internals/util"
 	"github.com/devtron-labs/git-sensor/pkg/git"
 	_ "github.com/robfig/cron/v3"
 	"go.uber.org/zap"
@@ -58,15 +58,15 @@ type RepoManagerImpl struct {
 	repositoryManagerAnalytics                    git.RepositoryManagerAnalytics
 	gitProviderRepository                         sql.GitProviderRepository
 	ciPipelineMaterialRepository                  sql.CiPipelineMaterialRepository
-	locker                                        *internal.RepositoryLocker
+	locker                                        *internals.RepositoryLocker
 	gitWatcher                                    git.GitWatcher
 	webhookEventRepository                        sql.WebhookEventRepository
 	webhookEventParsedDataRepository              sql.WebhookEventParsedDataRepository
 	webhookEventDataMappingRepository             sql.WebhookEventDataMappingRepository
 	webhookEventDataMappingFilterResultRepository sql.WebhookEventDataMappingFilterResultRepository
 	webhookEventBeanConverter                     git.WebhookEventBeanConverter
-	configuration                                 *internal.Configuration
-	gitManager                                    git.GitManagerImpl
+	configuration                                 *internals.Configuration
+	gitManager                                    git.GitManager
 }
 
 func NewRepoManagerImpl(
@@ -76,14 +76,14 @@ func NewRepoManagerImpl(
 	repositoryManagerAnalytics git.RepositoryManagerAnalytics,
 	gitProviderRepository sql.GitProviderRepository,
 	ciPipelineMaterialRepository sql.CiPipelineMaterialRepository,
-	locker *internal.RepositoryLocker,
+	locker *internals.RepositoryLocker,
 	gitWatcher git.GitWatcher, webhookEventRepository sql.WebhookEventRepository,
 	webhookEventParsedDataRepository sql.WebhookEventParsedDataRepository,
 	webhookEventDataMappingRepository sql.WebhookEventDataMappingRepository,
 	webhookEventDataMappingFilterResultRepository sql.WebhookEventDataMappingFilterResultRepository,
 	webhookEventBeanConverter git.WebhookEventBeanConverter,
-	configuration *internal.Configuration,
-	gitManager git.GitManagerImpl,
+	configuration *internals.Configuration,
+	gitManager git.GitManager,
 ) *RepoManagerImpl {
 	return &RepoManagerImpl{
 		logger:                            logger,
@@ -195,6 +195,9 @@ func (impl RepoManagerImpl) updatePipelineMaterialCommit(gitCtx git.GitContext,
 			continue
 		}
 
+		gitCtx = gitCtx.WithCredentials(material.GitProvider.UserName, material.GitProvider.Password).
+			WithCloningMode(impl.configuration.CloningMode)
+
 		commits, err := impl.repositoryManager.ChangesSince(gitCtx, material.CheckoutLocation, pipelineMaterial.Value, "", "", impl.configuration.GitHistoryCount)
 		//commits, err := impl.FetchChanges(pipelineMaterial.Id, "", "", 0)
 		if err == nil {
@@ -345,15 +348,18 @@ func (impl RepoManagerImpl) checkoutMaterial(gitCtx git.GitContext, material *sq
 	if err != nil {
 		return material, nil
 	}
-	checkoutPath, err := git.GetLocationForMaterial(material)
+
+	gitCtx = gitCtx.WithCredentials(userName, password).
+		WithCloningMode(impl.configuration.CloningMode)
+
+	checkoutPath, checkoutLocationForFetching, err := impl.repositoryManager.GetCheckoutPathAndLocation(gitCtx, material, gitProvider.Url)
 	if err != nil {
 		return material, err
 	}
-	gitCtx = gitCtx.WithCredentials(userName, password)
 
 	err = impl.repositoryManager.Add(gitCtx, material.GitProviderId, checkoutPath, material.Url, gitProvider.AuthMode, gitProvider.SshPrivateKey)
 	if err == nil {
-		material.CheckoutLocation = checkoutPath
+		material.CheckoutLocation = checkoutLocationForFetching
 		material.CheckoutStatus = true
 	} else {
 		material.CheckoutStatus = false
@@ -627,7 +633,10 @@ func (impl RepoManagerImpl) GetLatestCommitForBranch(gitCtx git.GitContext, pipe
 	}()
 
 	userName, password, err := git.GetUserNamePassword(gitMaterial.GitProvider)
-	gitCtx = gitCtx.WithCredentials(userName, password)
+
+	gitCtx = gitCtx.WithCredentials(userName, password).
+		WithCloningMode(impl.configuration.CloningMode)
+
 	updated, repo, err := impl.repositoryManager.Fetch(gitCtx, gitMaterial.Url, gitMaterial.CheckoutLocation)
 	if !updated {
 		impl.logger.Warn("repository is up to date")
@@ -659,7 +668,7 @@ func (impl RepoManagerImpl) GetLatestCommitForBranch(gitCtx git.GitContext, pipe
 		return nil, err
 	}
 
-	commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repo, branchName, "", "", 1)
+	commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repo, branchName, "", "", 1, gitMaterial.CheckoutLocation)
 
 	if commits == nil {
 		return nil, err
@@ -694,6 +703,8 @@ func (impl RepoManagerImpl) GetCommitMetadataForPipelineMaterial(gitCtx git.GitC
 		return nil, err
 	}
 
+	gitCtx = gitCtx.WithCredentials(gitMaterial.GitProvider.UserName, gitMaterial.GitProvider.Password).
+		WithCloningMode(impl.configuration.CloningMode)
 	// validate checkout status of gitMaterial
 	if !gitMaterial.CheckoutStatus {
 		impl.logger.Errorw("checkout not success", "gitMaterialId", gitMaterialId)
@@ -707,7 +718,6 @@ func (impl RepoManagerImpl) GetCommitMetadataForPipelineMaterial(gitCtx git.GitC
 		repoLock.Mutex.Unlock()
 		impl.locker.ReturnLocker(gitMaterial.Id)
 	}()
-
 	commits, err := impl.repositoryManager.ChangesSince(gitCtx, gitMaterial.CheckoutLocation, branchName, "", gitHash, 1)
 	if err != nil {
 		impl.logger.Errorw("error while fetching commit info", "pipelineMaterialId", pipelineMaterialId, "gitHash", gitHash, "err", err)
@@ -747,6 +757,10 @@ func (impl RepoManagerImpl) GetReleaseChanges(gitCtx git.GitContext, request *Re
 		repoLock.Mutex.Unlock()
 		impl.locker.ReturnLocker(gitMaterial.Id)
 	}()
+
+	gitCtx = gitCtx.WithCredentials(gitMaterial.GitProvider.UserName, gitMaterial.GitProvider.Password).
+		WithCloningMode(impl.configuration.CloningMode)
+
 	gitChanges, err := impl.repositoryManagerAnalytics.ChangesSinceByRepositoryForAnalytics(gitCtx, gitMaterial.CheckoutLocation, request.OldCommit, request.NewCommit)
 	if err != nil {
 		impl.logger.Errorw("error in computing changes", "req", request, "err", err)

pkg/git/Bean.go

@@ -17,7 +17,7 @@
 package git
 
 import (
-	"github.com/devtron-labs/git-sensor/internal/sql"
+	"github.com/devtron-labs/git-sensor/internals/sql"
 	"gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 	"io"

pkg/git/GitBaseManager.go

@@ -5,8 +5,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/devtron-labs/git-sensor/internal"
-	"github.com/devtron-labs/git-sensor/internal/sql"
+	"github.com/devtron-labs/git-sensor/internals"
+	"github.com/devtron-labs/git-sensor/internals/sql"
 	"github.com/devtron-labs/git-sensor/util"
 	"go.uber.org/zap"
 	"os"
@@ -29,8 +29,6 @@ type GitManager interface {
 	OpenRepoPlain(checkoutPath string) (*GitRepository, error)
 	// Init initializes a git repo
 	Init(gitCtx GitContext, rootDir string, remoteUrl string, isBare bool) error
-	// FetchDiffStatBetweenCommits returns the file stats reponse on executing git action
-	FetchDiffStatBetweenCommits(gitCtx GitContext, oldHash string, newHash string, rootDir string) (response, errMsg string, err error)
 }
 
 // GitManagerBase Base methods which will be available to all implementation of the parent interface
@@ -43,16 +41,21 @@ type GitManagerBase interface {
 	Checkout(gitCtx GitContext, rootDir, branch string) (response, errMsg string, err error)
 	// ConfigureSshCommand configures ssh in git repo
 	ConfigureSshCommand(gitCtx GitContext, rootDir string, sshPrivateKeyPath string) (response, errMsg string, err error)
+	//  FetchDiffStatBetweenCommits returns the file stats reponse on executing git action
+	FetchDiffStatBetweenCommits(gitCtx GitContext, oldHash string, newHash string, rootDir string) (response, errMsg string, err error)
 	// LogMergeBase get the commit diff between using a merge base strategy
 	LogMergeBase(gitCtx GitContext, rootDir, from string, to string) ([]*Commit, error)
+	CreateCmdWithContext(ctx GitContext, name string, arg ...string) (*exec.Cmd, context.CancelFunc)
+	RunCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error)
+	RunCommand(cmd *exec.Cmd) (response, errMsg string, err error)
 }
 type GitManagerBaseImpl struct {
 	logger            *zap.SugaredLogger
-	conf              *internal.Configuration
+	conf              *internals.Configuration
 	commandTimeoutMap map[string]int
 }
 
-func NewGitManagerBaseImpl(logger *zap.SugaredLogger, config *internal.Configuration) *GitManagerBaseImpl {
+func NewGitManagerBaseImpl(logger *zap.SugaredLogger, config *internals.Configuration) *GitManagerBaseImpl {
 
 	commandTimeoutMap, err := parseCmdTimeoutJson(config)
 	if err != nil {
@@ -62,7 +65,7 @@ func NewGitManagerBaseImpl(logger *zap.SugaredLogger, config *internal.Configura
 	return &GitManagerBaseImpl{logger: logger, conf: config, commandTimeoutMap: commandTimeoutMap}
 }
 
-func parseCmdTimeoutJson(config *internal.Configuration) (map[string]int, error) {
+func parseCmdTimeoutJson(config *internals.Configuration) (map[string]int, error) {
 	commandTimeoutMap := make(map[string]int)
 	var err error
 	if config.CliCmdTimeoutJson != "" {
@@ -75,41 +78,25 @@ type GitManagerImpl struct {
 	GitManager
 }
 
-func NewGitManagerImpl(configuration *internal.Configuration,
+func NewGitManagerImpl(configuration *internals.Configuration,
 	cliGitManager GitCliManager,
-	goGitManager GoGitSDKManager) GitManagerImpl {
+	goGitManager GoGitSDKManager) *GitManagerImpl {
 
 	if configuration.UseGitCli {
-		return GitManagerImpl{cliGitManager}
-	}
-	return GitManagerImpl{goGitManager}
-}
-
-func (impl *GitManagerImpl) OpenNewRepo(gitCtx GitContext, location string, url string) (*GitRepository, error) {
-
-	r, err := impl.OpenRepoPlain(location)
-	if err != nil {
-		err = os.RemoveAll(location)
-		if err != nil {
-			return r, fmt.Errorf("error in cleaning checkout path: %s", err)
-		}
-		err = impl.Init(gitCtx, location, url, true)
-		if err != nil {
-			return r, fmt.Errorf("err in git init: %s", err)
-		}
-		r, err = impl.OpenRepoPlain(location)
-		if err != nil {
-			return r, fmt.Errorf("err in git init: %s", err)
+		return &GitManagerImpl{
+			cliGitManager,
 		}
 	}
-	return r, nil
+	return &GitManagerImpl{
+		goGitManager,
+	}
 }
 
 func (impl *GitManagerBaseImpl) Fetch(gitCtx GitContext, rootDir string) (response, errMsg string, err error) {
 	impl.logger.Debugw("git fetch ", "location", rootDir)
 	cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "fetch", "origin", "--tags", "--force")
 	defer cancel()
-	output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password)
+	output, errMsg, err := impl.RunCommandWithCred(cmd, gitCtx.Username, gitCtx.Password)
 	impl.logger.Debugw("fetch output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
 	return output, errMsg, err
 }
@@ -118,7 +105,7 @@ func (impl *GitManagerBaseImpl) Checkout(gitCtx GitContext, rootDir, branch stri
 	impl.logger.Debugw("git checkout ", "location", rootDir)
 	cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "checkout", branch, "--force")
 	defer cancel()
-	output, errMsg, err := impl.runCommand(cmd)
+	output, errMsg, err := impl.RunCommand(cmd)
 	impl.logger.Debugw("checkout output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
 	return output, errMsg, err
 }
@@ -135,7 +122,7 @@ func (impl *GitManagerBaseImpl) LogMergeBase(gitCtx GitContext, rootDir, from st
 	impl.logger.Debugw("git", cmdArgs)
 	cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", cmdArgs...)
 	defer cancel()
-	output, errMsg, err := impl.runCommand(cmd)
+	output, errMsg, err := impl.RunCommand(cmd)
 	impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
 	if err != nil {
 		return nil, err
@@ -148,16 +135,16 @@ func (impl *GitManagerBaseImpl) LogMergeBase(gitCtx GitContext, rootDir, from st
 	return commits, nil
 }
 
-func (impl *GitManagerBaseImpl) runCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) {
+func (impl *GitManagerBaseImpl) RunCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) {
 	cmd.Env = append(os.Environ(),
 		fmt.Sprintf("GIT_ASKPASS=%s", GIT_ASK_PASS),
 		fmt.Sprintf("GIT_USERNAME=%s", userName),
 		fmt.Sprintf("GIT_PASSWORD=%s", password),
 	)
-	return impl.runCommand(cmd)
+	return impl.RunCommand(cmd)
 }
 
-func (impl *GitManagerBaseImpl) runCommand(cmd *exec.Cmd) (response, errMsg string, err error) {
+func (impl *GitManagerBaseImpl) RunCommand(cmd *exec.Cmd) (response, errMsg string, err error) {
 	cmd.Env = append(cmd.Env, "HOME=/dev/null")
 	outBytes, err := cmd.CombinedOutput()
 	if err != nil {
@@ -183,7 +170,7 @@ func (impl *GitManagerBaseImpl) ConfigureSshCommand(gitCtx GitContext, rootDir s
 	coreSshCommand := fmt.Sprintf("ssh -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", sshPrivateKeyPath)
 	cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "config", "core.sshCommand", coreSshCommand)
 	defer cancel()
-	output, errMsg, err := impl.runCommand(cmd)
+	output, errMsg, err := impl.RunCommand(cmd)
 	impl.logger.Debugw("configure ssh command output ", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
 	return output, errMsg, err
 }
@@ -280,7 +267,7 @@ func (impl *GitManagerBaseImpl) FetchDiffStatBetweenCommits(gitCtx GitContext, o
 	cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "diff", "--numstat", oldHash, newHash)
 	defer cancel()
 
-	output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password)
+	output, errMsg, err := impl.RunCommandWithCred(cmd, gitCtx.Username, gitCtx.Password)
 	impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
 	return output, errMsg, err
 }