From a36a72134d172e4355c684d6b8e13c88cb3800e8 Mon Sep 17 00:00:00 2001 From: rwliang Date: Sat, 11 Jul 2020 19:18:27 -0400 Subject: [PATCH] Git VCS driver: Support non-default target refs Currently, only the default ref (`master`) is supported. - Allow target non-default target refs to be specified via configs. - Add config-driven target ref auto-detection using the `HEAD` branch. - Update default repo base URL pattern to link to the content revision, accounting for potential non-default target ref usage. - Add global-level VCS configs which are merged into the repo-level configs during startup (repo-level vals take precedence). --- .gitignore | 2 + config-example.json | 16 +++++++- config/config.go | 73 +++++++++++++++++++++++++++----- config/config_test.go | 18 ++++++++ vcs/git.go | 96 ++++++++++++++++++++++++++++++++++++++----- vcs/git_test.go | 85 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 267 insertions(+), 23 deletions(-) create mode 100644 vcs/git_test.go diff --git a/.gitignore b/.gitignore index 129fe4de..da009bc4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ /node_modules .DS_Store *.exe +/db +/data diff --git a/config-example.json b/config-example.json index 64729e13..1a14324a 100644 --- a/config-example.json +++ b/config-example.json @@ -3,6 +3,11 @@ "dbpath" : "data", "title" : "Hound", "health-check-uri" : "/healthz", + "vcs-config" : { + "git": { + "detect-ref" : true + } + }, "repos" : { "SomeGitRepo" : { "url" : "https://www.github.com/YourOrganization/RepoOne.git" @@ -12,13 +17,19 @@ "ms-between-poll": 10000, "exclude-dot-files": true }, + "GitRepoWithDetectRefDisabled" : { + "url" : "https://www.github.com/YourOrganization/RepoOne.git", + "vcs-config" : { + "detect-ref" : false + } + }, "SomeMercurialRepo" : { "url" : "https://www.example.com/foo/hg", "vcs" : "hg" }, "Subversion" : { "url" : "http://my-svn.com/repo", - "url-pattern" : { + "url-pattern" : { "base-url" : "{url}/{path}{anchor}" }, "vcs" : "svn" @@ -49,6 +60,9 @@ "url-pattern" : { "base-url" : "https://{url}/src/master/{path}{anchor}", "anchor" : "#{filename}-{line}" + }, + "vcs-config" : { + "ref" : "main" } }, "RepoWithPollingDisabled" : { diff --git a/config/config.go b/config/config.go index db35900b..ee9abab3 100644 --- a/config/config.go +++ b/config/config.go @@ -14,7 +14,7 @@ const ( defaultPollEnabled = true defaultTitle = "Hound" defaultVcs = "git" - defaultBaseUrl = "{url}/blob/master/{path}{anchor}" + defaultBaseUrl = "{url}/blob/{rev}/{path}{anchor}" defaultAnchor = "#L{line}" defaultHealthCheckURI = "/healthz" ) @@ -55,11 +55,12 @@ func (r *Repo) PushUpdatesEnabled() bool { } type Config struct { - DbPath string `json:"dbpath"` - Title string `json:"title"` - Repos map[string]*Repo `json:"repos"` - MaxConcurrentIndexers int `json:"max-concurrent-indexers"` - HealthCheckURI string `json:"health-check-uri"` + DbPath string `json:"dbpath"` + Title string `json:"title"` + Repos map[string]*Repo `json:"repos"` + MaxConcurrentIndexers int `json:"max-concurrent-indexers"` + HealthCheckURI string `json:"health-check-uri"` + VCSConfigMessages map[string]*SecretMessage `json:"vcs-config"` } // SecretMessage is just like json.RawMessage but it will not @@ -116,8 +117,9 @@ func initRepo(r *Repo) { } } -// Populate missing config values with default values. -func initConfig(c *Config) { +// Populate missing config values with default values and +// merge global VCS configs into repo level configs. +func initConfig(c *Config) error { if c.MaxConcurrentIndexers == 0 { c.MaxConcurrentIndexers = defaultMaxConcurrentIndexers } @@ -125,6 +127,57 @@ func initConfig(c *Config) { if c.HealthCheckURI == "" { c.HealthCheckURI = defaultHealthCheckURI } + + return mergeVCSConfigs(c) +} + +func mergeVCSConfigs(cfg *Config) error { + globalConfigLen := len(cfg.VCSConfigMessages) + if globalConfigLen == 0 { + return nil + } + + globalConfigVals := make(map[string]map[string]interface{}, globalConfigLen) + for vcs, configBytes := range cfg.VCSConfigMessages { + var configVals map[string]interface{} + if err := json.Unmarshal(*configBytes, &configVals); err != nil { + return err + } + + globalConfigVals[vcs] = configVals + } + + for _, repo := range cfg.Repos { + var globalVals map[string]interface{} + globalVals, valsExist := globalConfigVals[repo.Vcs] + if !valsExist { + continue + } + + repoBytes := repo.VcsConfig() + var repoVals map[string]interface{} + if len(repoBytes) == 0 { + repoVals = make(map[string]interface{}, len(globalVals)) + } else if err := json.Unmarshal(repoBytes, &repoVals); err != nil { + return err + } + + for name, val := range globalVals { + if _, ok := repoVals[name]; !ok { + repoVals[name] = val + } + } + + repoBytes, err := json.Marshal(&repoVals) + if err != nil { + return err + } + + repoMessage := SecretMessage(repoBytes) + repo.VcsConfigMessage = &repoMessage + } + + return nil } func (c *Config) LoadFromFile(filename string) error { @@ -155,9 +208,7 @@ func (c *Config) LoadFromFile(filename string) error { initRepo(repo) } - initConfig(c) - - return nil + return initConfig(c) } func (c *Config) ToJsonString() (string, error) { diff --git a/config/config_test.go b/config/config_test.go index d1e70d95..2f867abb 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,6 +1,7 @@ package config import ( + "encoding/json" "path/filepath" "runtime" "testing" @@ -34,4 +35,21 @@ func TestExampleConfigsAreValid(t *testing.T) { t.Fatal(err) } } + + // Ensure that global VCS config vals are merged + repo := cfg.Repos["SomeGitRepo"] + vcsConfigBytes := repo.VcsConfig() + var vcsConfigVals map[string]interface{} + json.Unmarshal(vcsConfigBytes, &vcsConfigVals) + if detectRef, ok := vcsConfigVals["detect-ref"]; !ok || !detectRef.(bool) { + t.Error("global detectRef vcs config setting not set for repo") + } + + repo = cfg.Repos["GitRepoWithDetectRefDisabled"] + vcsConfigBytes = repo.VcsConfig() + json.Unmarshal(vcsConfigBytes, &vcsConfigVals) + if detectRef, ok := vcsConfigVals["detect-ref"]; !ok || detectRef.(bool) { + t.Error("global detectRef vcs config setting not overriden by repo-level setting") + } + } diff --git a/vcs/git.go b/vcs/git.go index f8c66825..bc7ec745 100644 --- a/vcs/git.go +++ b/vcs/git.go @@ -2,24 +2,49 @@ package vcs import ( "bytes" + "encoding/json" "fmt" "io" "log" "os/exec" "path/filepath" + "regexp" "strings" ) const defaultRef = "master" +var headBranchRegexp = regexp.MustCompile(`HEAD branch: (?P.+)`) + func init() { Register(newGit, "git") } -type GitDriver struct{} +type GitDriver struct { + DetectRef bool `json:"detect-ref"` + Ref string `json:"ref"` + refDetetector refDetetector +} + +type refDetetector interface { + detectRef(dir string) string +} + +type headBranchDetector struct { +} func newGit(b []byte) (Driver, error) { - return &GitDriver{}, nil + var d GitDriver + + if b != nil { + if err := json.Unmarshal(b, &d); err != nil { + return nil, err + } + } + + d.refDetetector = &headBranchDetector{} + + return &d, nil } func (g *GitDriver) HeadRev(dir string) (string, error) { @@ -47,43 +72,61 @@ func (g *GitDriver) HeadRev(dir string) (string, error) { return strings.TrimSpace(buf.String()), cmd.Wait() } -func run(desc, dir, cmd string, args ...string) error { +func run(desc, dir, cmd string, args ...string) (string, error) { c := exec.Command(cmd, args...) c.Dir = dir - if out, err := c.CombinedOutput(); err != nil { + out, err := c.CombinedOutput() + if err != nil { log.Printf( "Failed to %s %s, see output below\n%sContinuing...", desc, dir, out) - return err } - return nil + + return string(out), nil } func (g *GitDriver) Pull(dir string) (string, error) { - if err := run("git fetch", dir, + targetRef := g.targetRef(dir) + + if _, err := run("git fetch", dir, "git", "fetch", "--prune", "--no-tags", "--depth", "1", "origin", - fmt.Sprintf("+%s:remotes/origin/%s", defaultRef, defaultRef)); err != nil { + fmt.Sprintf("+%s:remotes/origin/%s", targetRef, targetRef)); err != nil { return "", err } - if err := run("git reset", dir, + if _, err := run("git reset", dir, "git", "reset", "--hard", - fmt.Sprintf("origin/%s", defaultRef)); err != nil { + fmt.Sprintf("origin/%s", targetRef)); err != nil { return "", err } return g.HeadRev(dir) } +func (g *GitDriver) targetRef(dir string) string { + var targetRef string + if g.Ref != "" { + targetRef = g.Ref + } else if g.DetectRef { + targetRef = g.refDetetector.detectRef(dir) + } + + if len(targetRef) == 0 { + targetRef = defaultRef + } + + return targetRef +} + func (g *GitDriver) Clone(dir, url string) (string, error) { par, rep := filepath.Split(dir) cmd := exec.Command( @@ -99,7 +142,7 @@ func (g *GitDriver) Clone(dir, url string) (string, error) { return "", err } - return g.HeadRev(dir) + return g.Pull(dir) } func (g *GitDriver) SpecialFiles() []string { @@ -107,3 +150,34 @@ func (g *GitDriver) SpecialFiles() []string { ".git", } } + +func (d *headBranchDetector) detectRef(dir string) string { + output, err := run("git show remote info", dir, + "git", + "remote", + "show", + "origin", + ) + + if err != nil { + log.Printf( + "error occured when fetching info to determine target ref in %s: %s. Will fall back to default ref %s", + dir, + err, + defaultRef, + ) + return "" + } + + matches := headBranchRegexp.FindStringSubmatch(output) + if len(matches) != 2 { + log.Printf( + "could not determine target ref in %s. Will fall back to default ref %s", + dir, + defaultRef, + ) + return "" + } + + return matches[1] +} diff --git a/vcs/git_test.go b/vcs/git_test.go new file mode 100644 index 00000000..56016487 --- /dev/null +++ b/vcs/git_test.go @@ -0,0 +1,85 @@ +package vcs + +import ( + "fmt" + "testing" +) + +type testRefDetector struct { + result string +} + +func (d *testRefDetector) detectRef(dir string) string { + return d.result +} + +func TestTargetRef(t *testing.T) { + testCases := []struct { + explicitRef string + detectRefEnabled bool + detectRefResult string + expectedResult string + }{ + { + explicitRef: "", + detectRefEnabled: true, + detectRefResult: "detected-ref", + expectedResult: "detected-ref", + }, + { + explicitRef: "", + detectRefEnabled: true, + detectRefResult: "", + expectedResult: defaultRef, + }, + { + explicitRef: "explicit-ref", + detectRefEnabled: true, + detectRefResult: "detected-ref", + expectedResult: "explicit-ref", + }, + { + explicitRef: "explicit-ref", + detectRefEnabled: true, + detectRefResult: "", + expectedResult: "explicit-ref", + }, + { + explicitRef: "explicit-ref", + detectRefEnabled: false, + detectRefResult: "foo", + expectedResult: "explicit-ref", + }, + { + explicitRef: "", + detectRefEnabled: false, + detectRefResult: "", + expectedResult: defaultRef, + }, + { + explicitRef: "explicit-ref", + detectRefEnabled: false, + detectRefResult: "", + expectedResult: "explicit-ref", + }, + { + explicitRef: "explicit-ref", + detectRefEnabled: false, + detectRefResult: "detected-ref", + expectedResult: "explicit-ref", + }, + } + for idx, testCase := range testCases { + t.Run(fmt.Sprintf("test case %d", idx), func(t *testing.T) { + driver := &GitDriver{ + Ref: testCase.explicitRef, + DetectRef: testCase.detectRefEnabled, + refDetetector: &testRefDetector{result: testCase.detectRefResult}, + } + actualResult := driver.targetRef("dir") + if actualResult != testCase.expectedResult { + t.Errorf("expected target ref: %q, got: %q", testCase.expectedResult, testCase.actualResult) + } + }) + } +}