Marged refactoring

go.mod

@@ -5,8 +5,9 @@ go 1.20
 require (
 	github.com/caarlos0/env v3.5.0+incompatible
 	github.com/devtron-labs/common-lib v0.0.10-0.20240118140811-762cb3b7f9a4
-	github.com/devtron-labs/protos v0.0.2
+	github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb
 	github.com/gammazero/workerpool v0.0.0-20200206003619-019d125201ab
+	github.com/go-git/go-git/v5 v5.11.0
 	github.com/go-pg/pg v6.15.1+incompatible
 	github.com/google/wire v0.4.0
 	github.com/gorilla/handlers v1.4.2
@@ -22,48 +23,59 @@ require (
 	google.golang.org/grpc v1.56.3
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/src-d/go-billy.v4 v4.3.2
-	gopkg.in/src-d/go-git.v4 v4.13.1
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
 	github.com/arl/statsviz v0.6.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/gammazero/deque v0.0.0-20200124200322-7e84b94275b8 // indirect
+	github.com/gliderlabs/ssh v0.3.6 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-git/go-billy/v5 v5.5.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hpcloud/tail v1.0.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
-	github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
-	github.com/kr/pretty v0.2.1 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/highwayhash v1.0.2 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/nats-io/jwt/v2 v2.3.0 // indirect
 	github.com/nats-io/nats.go v1.28.0 // indirect
 	github.com/nats-io/nkeys v0.4.6 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/onsi/ginkgo v1.10.3 // indirect
-	github.com/onsi/gomega v1.7.1 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
 	github.com/prometheus/common v0.38.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
-	github.com/src-d/gcfg v1.4.0 // indirect
+	github.com/skeema/knownhosts v1.2.1 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
-	github.com/xanzy/ssh-agent v0.2.1 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.13.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
+	gopkg.in/fsnotify.v1 v1.4.7 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	mellium.im/sasl v0.3.1 // indirect

pkg/RepoManages.go

@@ -198,11 +198,16 @@ func (impl RepoManagerImpl) updatePipelineMaterialCommit(gitCtx git.GitContext,
 		gitCtx = gitCtx.WithCredentials(material.GitProvider.UserName, material.GitProvider.Password).
 			WithCloningMode(impl.configuration.CloningMode)
 
-		commits, err := impl.repositoryManager.ChangesSince(gitCtx, material.CheckoutLocation, pipelineMaterial.Value, "", "", impl.configuration.GitHistoryCount)
+		fetchCount := impl.configuration.GitHistoryCount
+		commits, err := impl.repositoryManager.ChangesSince(gitCtx, material.CheckoutLocation, pipelineMaterial.Value, "", "", fetchCount)
 		//commits, err := impl.FetchChanges(pipelineMaterial.Id, "", "", 0)
 		if err == nil {
 			impl.logger.Infow("commits found", "commit", commits)
-			b, err := json.Marshal(commits)
+			totalCommits, err := git.AppendOldCommitsFromHistory(commits, pipelineMaterial.CommitHistory, fetchCount)
+			if err != nil {
+				impl.logger.Errorw("error in appending history to fetched commit", "err", err, "pipelineMaterialId", pipelineMaterial.Id)
+			}
+			b, err := json.Marshal(totalCommits)
 			if err == nil {
 				pipelineMaterial.CommitHistory = string(b)
 				if len(commits) > 0 {

pkg/git/Bean.go

@@ -17,9 +17,11 @@
 package git
 
 import (
+	"encoding/json"
+	"fmt"
 	"github.com/devtron-labs/git-sensor/internals/sql"
-	"gopkg.in/src-d/go-git.v4"
-	"gopkg.in/src-d/go-git.v4/plumbing/object"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/object"
 	"io"
 	"time"
 	"unicode/utf8"
@@ -141,6 +143,22 @@ type GitCommitBase struct {
 	Excluded    bool         `json:",omitempty"`
 }
 
+func AppendOldCommitsFromHistory(newCommits []*GitCommitBase, commitHistory string, fetchedCount int) ([]*GitCommitBase, error) {
+
+	oldCommits := make([]*GitCommitBase, 0)
+	if len(commitHistory) > 0 {
+		err := json.Unmarshal([]byte(commitHistory), &oldCommits)
+		if err != nil {
+			return newCommits, fmt.Errorf("unmarshalling error %v", err)
+		}
+	}
+	totalCommits := append(newCommits, oldCommits...)
+	if len(totalCommits) > fetchedCount {
+		totalCommits = totalCommits[:fetchedCount]
+	}
+	return totalCommits, nil
+}
+
 func (gitCommit *GitCommitBase) SetFileStats(stats *FileStats) {
 	gitCommit.FileStats = stats
 }

pkg/git/GitCliManager.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 )
 
 type GitCliManagerImpl struct {
@@ -165,11 +166,18 @@ func (impl *GitCliManagerImpl) processGitLogOutput(out string) ([]GitCommit, err
 
 	for _, formattedCommit := range gitCommitFormattedList {
 
+		subject := strings.TrimSpace(formattedCommit.Subject)
+		body := strings.TrimSpace(formattedCommit.Body)
+		message := subject
+		if len(body) > 0 {
+			message = strings.Join([]string{subject, body}, "\n")
+		}
+
 		cm := GitCommitBase{
 			Commit:  formattedCommit.Commit,
 			Author:  formattedCommit.Commiter.Name + " <" + formattedCommit.Commiter.Email + ">",
 			Date:    formattedCommit.Commiter.Date,
-			Message: formattedCommit.Subject + "\n" + formattedCommit.Body,
+			Message: message,
 		}
 		gitCommits = append(gitCommits, &GitCommitCli{
 			GitCommitBase: cm,

pkg/git/GitFormatter.go

@@ -2,7 +2,6 @@ package git
 
 import (
 	"encoding/json"
-	"fmt"
 	"strconv"
 	"strings"
 	"time"
@@ -17,8 +16,8 @@ var GITFORMAT = "--pretty=format:{" +
 	_dl_ + "commit" + _dl_ + ":" + _dl_ + "%H" + _dl_ + "," +
 	_dl_ + "parent" + _dl_ + ":" + _dl_ + "%P" + _dl_ + "," +
 	_dl_ + "refs" + _dl_ + ":" + _dl_ + "%D" + _dl_ + "," +
-	_dl_ + "subject" + _dl_ + ":" + _dl_ + "%s" + _dl_ + "," +
-	_dl_ + "body" + _dl_ + ":" + _dl_ + "%b" + _dl_ + "," +
+	_dl_ + "subject" + _dl_ + ":" + _dl_ + "%<(1024,trunc)%s" + _dl_ + "," +
+	_dl_ + "body" + _dl_ + ":" + _dl_ + "%<(1024,trunc)%b" + _dl_ + "," +
 	_dl_ + "author" + _dl_ +
 	":{" +
 	_dl_ + "name" + _dl_ + ":" + _dl_ + "%aN" + _dl_ + "," +
@@ -52,13 +51,16 @@ func parseFormattedLogOutput(out string) ([]GitCommitFormat, error) {
 	out = strings.ReplaceAll(out, "},\n", "},")
 
 	// to escape the special characters like quotes and newline characters in the commit data
-	logOut := strconv.Quote(out)
+	var sb strings.Builder
+	buffer := strconv.AppendQuote(make([]byte, 0, len(out)), out)
+	sb.Write(buffer)
+	logOut := sb.String()
 
 	//replace the delimiter with quotes to make it parsable json
 	logOut = strings.ReplaceAll(logOut, _dl_, `"`)
 
-	logOut = logOut[1 : len(logOut)-2]   // trim surround characters (surrounding quotes and trailing comma)
-	logOut = fmt.Sprintf("[%s]", logOut) // Add []
+	logOut = logOut[1 : len(logOut)-2] // trim surround characters (surrounding quotes and trailing comma)
+	logOut = strings.Join([]string{"[", "]"}, logOut)
 
 	var gitCommitFormattedList []GitCommitFormat
 	err := json.Unmarshal([]byte(logOut), &gitCommitFormattedList)
@@ -85,7 +87,7 @@ func (formattedCommit GitCommitFormat) transformToCommit() *Commit {
 		},
 		Tag:     &Tag{},
 		Tree:    &Tree{},
-		Subject: formattedCommit.Subject,
-		Body:    formattedCommit.Body,
+		Subject: strings.TrimSpace(formattedCommit.Subject),
+		Body:    strings.TrimSpace(formattedCommit.Body),
 	}
 }

pkg/git/GoGitSDKManager.go

@@ -2,10 +2,10 @@ package git
 
 import (
 	"fmt"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
+	"github.com/go-git/go-git/v5/plumbing"
 	"go.uber.org/zap"
-	"gopkg.in/src-d/go-git.v4"
-	"gopkg.in/src-d/go-git.v4/config"
-	"gopkg.in/src-d/go-git.v4/plumbing"
 	"os"
 )
 

pkg/git/RepositoryManagerAnalytics.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	"github.com/devtron-labs/git-sensor/internals"
 	"github.com/devtron-labs/git-sensor/util"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
 	"go.uber.org/zap"
-	"gopkg.in/src-d/go-git.v4"
-	"gopkg.in/src-d/go-git.v4/plumbing"
-	"gopkg.in/src-d/go-git.v4/plumbing/object"
 	"io"
 	"log"
 	"strings"

pkg/git/Watcher.go

@@ -240,14 +240,26 @@ func (impl GitWatcherImpl) pollGitMaterialAndNotify(material *sql.GitMaterial) e
 		impl.logger.Debugw("Running changesBySinceRepository for material - ", material)
 		impl.logger.Debugw("---------------------------------------------------------- ")
 		// parse env variables here, then search for the count field and pass here.
-		commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repo, material.Value, "", "", impl.configuration.GitHistoryCount, checkoutLocation)
+		lastSeenHash := ""
+		if len(material.LastSeenHash) > 0 {
+			// this might misbehave is the hash stored in table is corrupted somehow
+			lastSeenHash = material.LastSeenHash
+		}
+		fetchCount := impl.configuration.GitHistoryCount
+		commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repo, material.Value, lastSeenHash, "", fetchCount, checkoutLocation)
 		if err != nil {
 			material.Errored = true
 			material.ErrorMsg = err.Error()
 			erroredMaterialsModels = append(erroredMaterialsModels, material)
 		} else if len(commits) > 0 {
 			latestCommit := commits[0]
 			if latestCommit.GetCommit().Commit != material.LastSeenHash {
+
+				commitsTotal, err := AppendOldCommitsFromHistory(commits, material.CommitHistory, fetchCount)
+				if err != nil {
+					impl.logger.Errorw("error in appending history to new commits", "material", material.GitMaterialId, "err", err)
+				}
+
 				// new commit found
 				mb := &CiPipelineMaterialBean{
 					Id:            material.Id,
@@ -263,7 +275,7 @@ func (impl GitWatcherImpl) pollGitMaterialAndNotify(material *sql.GitMaterial) e
 				material.CommitAuthor = latestCommit.Author
 				material.CommitDate = latestCommit.Date
 				material.CommitMessage = latestCommit.Message
-				commitJson, _ := json.Marshal(commits)
+				commitJson, _ := json.Marshal(commitsTotal)
 				material.CommitHistory = string(commitJson)
 				material.Errored = false
 				material.ErrorMsg = ""

vendor/dario.cat/mergo/.deepsource.toml

@@ -0,0 +1,12 @@
+version = 1
+
+test_patterns = [
+  "*_test.go"
+]
+
+[[analyzers]]
+name = "go"
+enabled = true
+
+  [analyzers.meta]
+  import_path = "dario.cat/mergo"

vendor/dario.cat/mergo/.gitignore

@@ -0,0 +1,33 @@
+#### joe made this: http://goel.io/joe
+
+#### go ####
+# Binaries for programs and plugins
+*.exe
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
+.glide/
+
+#### vim ####
+# Swap
+[._]*.s[a-v][a-z]
+[._]*.sw[a-p]
+[._]s[a-v][a-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags

vendor/dario.cat/mergo/.travis.yml

@@ -0,0 +1,12 @@
+language: go
+arch:
+    - amd64
+    - ppc64le
+install:
+  - go get -t
+  - go get golang.org/x/tools/cmd/cover
+  - go get github.com/mattn/goveralls
+script:
+  - go test -race -v ./...
+after_script:
+  - $HOME/gopath/bin/goveralls -service=travis-ci -repotoken $COVERALLS_TOKEN

vendor/dario.cat/mergo/CODE_OF_CONDUCT.md

@@ -0,0 +1,46 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at [email protected]. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/