From 4bdc1e7e97a477e99957e213280ef2c767761e89 Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Tue, 20 Dec 2022 13:30:46 -0500 Subject: [PATCH 01/15] adding support for public and private git providers Signed-off-by: Michael Hoang --- pkg/devfile/parser/parse.go | 20 +- pkg/devfile/parser/parse_test.go | 51 ++-- pkg/util/git.go | 276 +++++++++++++++++++ pkg/util/git_test.go | 453 +++++++++++++++++++++++++++++++ pkg/util/util.go | 54 ---- pkg/util/util_test.go | 79 ------ 6 files changed, 768 insertions(+), 165 deletions(-) create mode 100644 pkg/util/git.go create mode 100644 pkg/util/git_test.go diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index 5523d39b..42c17e5c 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -425,15 +425,17 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D } d.Ctx = devfileCtx.NewURLDevfileCtx(newUri) - if strings.Contains(newUri, "raw.githubusercontent.com") { - urlComponents, err := util.GetGitUrlComponentsFromRaw(newUri) + if strings.Contains(newUri, util.RawGitHubHost) { + gitUrl, err := util.ParseGitUrl(newUri) if err != nil { return DevfileObj{}, err } - destDir := path.Dir(curDevfileCtx.GetAbsPath()) - err = getResourcesFromGit(urlComponents, destDir) - if err != nil { - return DevfileObj{}, err + if gitUrl.IsFile { + destDir := path.Dir(curDevfileCtx.GetAbsPath()) + err = getResourcesFromGit(gitUrl, destDir) + if err != nil { + return DevfileObj{}, err + } } } } @@ -443,19 +445,19 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D return populateAndParseDevfile(d, newResolveCtx, tool, true) } -func getResourcesFromGit(gitUrlComponents map[string]string, destDir string) error { +func getResourcesFromGit(g util.GitUrl, destDir string) error { stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) if err != nil { return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) } defer os.RemoveAll(stackDir) - err = util.CloneGitRepo(gitUrlComponents, stackDir) + err = util.CloneGitRepo(g, stackDir) if err != nil { return err } - dir := path.Dir(path.Join(stackDir, gitUrlComponents["file"])) + dir := path.Dir(path.Join(stackDir, g.Path)) err = util.CopyAllDirFiles(dir, destDir) if err != nil { return err diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index f80aaa4e..50223bad 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -18,6 +18,7 @@ package parser import ( "context" "fmt" + "github.com/devfile/library/v2/pkg/util" "io/ioutil" "net" "net/http" @@ -4165,42 +4166,46 @@ func Test_getResourcesFromGit(t *testing.T) { } defer os.RemoveAll(destDir) - invalidGitUrl := map[string]string{ - "username": "devfile", - "project": "nonexistent", - "branch": "nonexistent", + invalidGitHubUrl := util.GitUrl{ + Protocol: "", + Host: "", + Owner: "devfile", + Repo: "nonexistent", + Branch: "nonexistent", } - validGitUrl := map[string]string{ - "host": "raw.githubusercontent.com", - "username": "devfile", - "project": "registry", - "branch": "main", - "file": "stacks/nodejs/devfile.yaml", + validGitHubUrl := util.GitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "registry", + Branch: "main", + Path: "stacks/nodejs/devfile.yaml", + IsFile: true, } tests := []struct { - name string - gitUrlComponents map[string]string - destDir string - wantErr bool + name string + gitUrl util.GitUrl + destDir string + wantErr bool }{ { - name: "should fail with invalid git url", - gitUrlComponents: invalidGitUrl, - destDir: path.Join(os.TempDir(), "nonexistent"), - wantErr: true, + name: "should fail with invalid git url", + gitUrl: invalidGitHubUrl, + destDir: path.Join(os.TempDir(), "nonexistent"), + wantErr: true, }, { - name: "should be able to get resources from valid git url", - gitUrlComponents: validGitUrl, - destDir: destDir, - wantErr: false, + name: "should be able to get resources from valid git url", + gitUrl: validGitHubUrl, + destDir: destDir, + wantErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := getResourcesFromGit(tt.gitUrlComponents, tt.destDir) + err := getResourcesFromGit(tt.gitUrl, tt.destDir) if (err != nil) != tt.wantErr { t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err) } diff --git a/pkg/util/git.go b/pkg/util/git.go new file mode 100644 index 00000000..bbf7be8d --- /dev/null +++ b/pkg/util/git.go @@ -0,0 +1,276 @@ +// +// Copyright 2022 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "fmt" + gitpkg "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" +) + +const ( + GitHubHost string = "github.com" + RawGitHubHost string = "raw.githubusercontent.com" + GitLabHost string = "gitlab.com" + BitbucketHost string = "bitbucket.org" + + GitHubToken string = "GITHUB_TOKEN" + GitLabToken string = "GITLAB_TOKEN" + BitbucketToken string = "BITBUCKET_TOKEN" +) + +type GitUrl struct { + Protocol string + Host string + Owner string + Repo string + Branch string + Path string + token string + IsFile bool +} + +// ParseGitUrl extracts information from a GitHub, GitLab, or Bitbucket url +// A client is used to check whether the url is private or public, and sets +// the providers personal access token from the environment variable +func ParseGitUrl(fullUrl string) (GitUrl, error) { + var c = http.Client{ + Timeout: HTTPRequestResponseTimeout, + } + return parseGitUrlWithClient(fullUrl, c) +} + +func parseGitUrlWithClient(fullUrl string, c http.Client) (GitUrl, error) { + var g GitUrl + + err := ValidateURL(fullUrl) + if err != nil { + return g, err + } + + parsedUrl, err := url.Parse(fullUrl) + if err != nil { + return g, err + } + + if len(parsedUrl.Path) == 0 { + return g, fmt.Errorf("url path should not be empty") + } + + if parsedUrl.Host == RawGitHubHost || parsedUrl.Host == GitHubHost { + g, err = parseGitHubUrl(g, parsedUrl, c) + } else if parsedUrl.Host == GitLabHost { + g, err = parseGitLabUrl(g, parsedUrl, c) + } else if parsedUrl.Host == BitbucketHost { + g, err = parseBitbucketUrl(g, parsedUrl, c) + } else { + err = fmt.Errorf("url host should be a valid GitHub, GitLab, or Bitbucket host; received: %s", parsedUrl.Host) + } + + return g, err +} + +func parseGitHubUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { + var splitUrl []string + var err error + + g.Protocol = url.Scheme + g.Host = url.Host + + if g.Host == RawGitHubHost { + g.IsFile = true + // raw GitHub urls don't contain "blob" or "tree" + splitUrl = strings.SplitN(url.Path[1:], "/", 4) + if len(splitUrl) == 4 { + g.Owner = splitUrl[0] + g.Repo = splitUrl[1] + g.Branch = splitUrl[2] + g.Path = splitUrl[3] + } else { + err = fmt.Errorf("raw url path should contain ///, received: %s", url.Path[1:]) + } + } + + if g.Host == GitHubHost { + splitUrl = strings.SplitN(url.Path[1:], "/", 5) + if len(splitUrl) < 2 { + err = fmt.Errorf("url path should contain /, received: %s", url.Path[1:]) + } else { + g.Owner = splitUrl[0] + g.Repo = splitUrl[1] + + if len(splitUrl) == 5 { + switch splitUrl[2] { + case "tree": + g.IsFile = false + case "blob": + g.IsFile = true + } + g.Branch = splitUrl[3] + g.Path = splitUrl[4] + } + } + } + + if !isGitUrlPublic(g, c) { + g.token = os.Getenv(GitHubToken) + } + + return g, err +} + +func parseGitLabUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { + var splitFile, splitOrg []string + var err error + + g.Protocol = url.Scheme + g.Host = url.Host + g.IsFile = false + + // GitLab urls contain a '-' separating the root of the repo + // and the path to a file or directory + split := strings.Split(url.Path[1:], "/-/") + + splitOrg = strings.SplitN(split[0], "/", 2) + if len(split) == 2 { + splitFile = strings.SplitN(split[1], "/", 3) + } + + if len(splitOrg) < 2 { + err = fmt.Errorf("url path should contain /, received: %s", url.Path[1:]) + } else { + g.Owner = splitOrg[0] + g.Repo = splitOrg[1] + } + + if len(splitFile) == 3 { + if splitFile[0] == "blob" || splitFile[0] == "tree" || splitFile[0] == "raw" { + g.Branch = splitFile[1] + g.Path = splitFile[2] + ext := filepath.Ext(g.Path) + if ext != "" { + g.IsFile = true + } + } else { + err = fmt.Errorf("url path should contain 'blob' or 'tree' or 'raw', received: %s", url.Path[1:]) + } + } + + if !isGitUrlPublic(g, c) { + g.token = os.Getenv(GitLabToken) + } + + return g, err +} + +func parseBitbucketUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { + var splitUrl []string + var err error + + g.Protocol = url.Scheme + g.Host = url.Host + g.IsFile = false + + splitUrl = strings.SplitN(url.Path[1:], "/", 5) + if len(splitUrl) < 2 { + err = fmt.Errorf("url path should contain /, received: %s", url.Path[1:]) + } else if len(splitUrl) == 2 { + g.Owner = splitUrl[0] + g.Repo = splitUrl[1] + } else { + g.Owner = splitUrl[0] + g.Repo = splitUrl[1] + if len(splitUrl) == 5 { + if splitUrl[2] == "raw" || splitUrl[2] == "src" { + g.Branch = splitUrl[3] + g.Path = splitUrl[4] + ext := filepath.Ext(g.Path) + if ext != "" { + g.IsFile = true + } + } else { + err = fmt.Errorf("url path should contain 'raw' or 'src', received: %s", url.Path[1:]) + } + } else { + err = fmt.Errorf("url path should contain path to directory or file, received: %s", url.Path[1:]) + } + } + + if !isGitUrlPublic(g, c) { + g.token = os.Getenv(BitbucketToken) + } + + return g, err +} + +func isGitUrlPublic(g GitUrl, c http.Client) bool { + host := g.Host + if host == RawGitHubHost { + host = GitHubHost + } + + repo := fmt.Sprintf("%s://%s/%s/%s", g.Protocol, host, g.Owner, g.Repo) + + if res, err := c.Get(repo); err != nil { + return false + } else if res.StatusCode == http.StatusOK { + return true + } + return false +} + +// CloneGitRepo clones a GitHub Repo to a destination directory +func CloneGitRepo(g GitUrl, destDir string) error { + var cloneOptions *gitpkg.CloneOptions + + host := g.Host + if host == RawGitHubHost { + host = GitHubHost + } + + repoUrl := fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, host, g.Owner, g.Repo) + branch := fmt.Sprintf("refs/heads/%s", g.Branch) + + cloneOptions = &gitpkg.CloneOptions{ + URL: repoUrl, + ReferenceName: plumbing.ReferenceName(branch), + SingleBranch: true, + Depth: 1, + } + + if g.token != "" { + cloneOptions.Auth = &githttp.BasicAuth{ + // go-git auth allows username to be anything except + // an empty string for GitHub and GitLab, however requires + // for Bitbucket to be "x-token-auth" + Username: "x-token-auth", + Password: g.token, + } + } + + _, err := gitpkg.PlainClone(destDir, false, cloneOptions) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/util/git_test.go b/pkg/util/git_test.go new file mode 100644 index 00000000..2d62cc26 --- /dev/null +++ b/pkg/util/git_test.go @@ -0,0 +1,453 @@ +// +// Copyright 2021-2022 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "github.com/kylelemons/godebug/pretty" + "github.com/stretchr/testify/assert" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "reflect" + "testing" +) + +var ( + githubToken = "fake-github-token" + gitlabToken = "fake-gitlab-token" + bitbucketToken = "fake-bitbucket-token" +) + +type respondWithStatus struct { + status int +} + +func (rs respondWithStatus) RoundTrip(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: rs.status, + }, nil +} + +var ( + publicClient = http.Client{Transport: respondWithStatus{status: http.StatusOK}} + privateClient = http.Client{Transport: respondWithStatus{status: http.StatusNotFound}} +) + +func Test_parseGitUrlWithClient(t *testing.T) { + defer func() { + err := os.Unsetenv(githubToken) + if err != nil { + t.Errorf("Failed to unset GitHub token") + } + err = os.Unsetenv(gitlabToken) + if err != nil { + t.Errorf("Failed to unset GitLab token") + } + err = os.Unsetenv(bitbucketToken) + if err != nil { + t.Errorf("Failed to unset Bitbucket token") + } + }() + + err := os.Setenv("GITHUB_TOKEN", githubToken) + if err != nil { + t.Errorf("Failed to set GitHub token") + } + err = os.Setenv("GITLAB_TOKEN", gitlabToken) + if err != nil { + t.Errorf("Failed to set GitLab token") + } + err = os.Setenv("BITBUCKET_TOKEN", bitbucketToken) + if err != nil { + t.Errorf("Failed to set Bitbucket token") + } + + tests := []struct { + name string + url string + client http.Client + wantUrl GitUrl + wantErr string + }{ + { + name: "should fail with empty url", + url: "", + client: publicClient, + wantErr: "URL is invalid", + }, + { + name: "should fail with invalid git host", + url: "https://google.ca/", + client: publicClient, + wantErr: "url host should be a valid GitHub, GitLab, or Bitbucket host*", + }, + // GitHub + { + name: "should parse public GitHub repo with root path", + url: "https://github.com/devfile/library", + client: publicClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Branch: "", + Path: "", + token: "", + IsFile: false, + }, + }, + { + name: "should fail with only GitHub host", + url: "https://github.com/", + client: publicClient, + wantErr: "url path should contain /*", + }, + { + name: "should parse public GitHub repo with file path", + url: "https://github.com/devfile/library/blob/main/devfile.yaml", + client: publicClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Branch: "main", + Path: "devfile.yaml", + token: "", + IsFile: true, + }, + }, + { + name: "should parse public GitHub repo with raw file path", + url: "https://raw.githubusercontent.com/devfile/library/main/devfile.yaml", + client: publicClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "library", + Branch: "main", + Path: "devfile.yaml", + token: "", + IsFile: true, + }, + }, + { + name: "should fail with missing GitHub repo", + url: "https://github.com/devfile", + client: publicClient, + wantErr: "url path should contain /*", + }, + { + name: "should fail with invalid GitHub raw file path", + url: "https://raw.githubusercontent.com/devfile/library/devfile.yaml", + client: publicClient, + wantErr: "raw url path should contain ///*", + }, + { + name: "should parse private GitHub repo with token", + url: "https://github.com/fake-owner/fake-private-repo", + client: privateClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "fake-owner", + Repo: "fake-private-repo", + Branch: "", + Path: "", + token: "fake-github-token", + IsFile: false, + }, + }, + { + name: "should parse private raw GitHub file path with token", + url: "https://raw.githubusercontent.com/fake-owner/fake-private-repo/main/README.md", + client: privateClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "fake-owner", + Repo: "fake-private-repo", + Branch: "main", + Path: "README.md", + token: "fake-github-token", + IsFile: true, + }, + }, + // Gitlab + { + name: "should parse public GitLab repo with root path", + url: "https://gitlab.com/gitlab-org/gitlab-foss", + client: publicClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "gitlab.com", + Owner: "gitlab-org", + Repo: "gitlab-foss", + Branch: "", + Path: "", + token: "", + IsFile: false, + }, + }, + { + name: "should fail with only GitLab host", + url: "https://gitlab.com/", + client: publicClient, + wantErr: "url path should contain /*", + }, + { + name: "should parse public GitLab repo with file path", + url: "https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/README.md", + client: publicClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "gitlab.com", + Owner: "gitlab-org", + Repo: "gitlab-foss", + Branch: "master", + Path: "README.md", + token: "", + IsFile: true, + }, + }, + { + name: "should fail with missing GitLab repo", + url: "https://gitlab.com/gitlab-org", + client: publicClient, + wantErr: "url path should contain /*", + }, + { + name: "should fail with missing GitLab keywords", + url: "https://gitlab.com/gitlab-org/gitlab-foss/-/master/directory/README.md", + client: publicClient, + wantErr: "url path should contain 'blob' or 'tree' or 'raw'*", + }, + { + name: "should parse private GitLab repo with token", + url: "https://gitlab.com/fake-owner/fake-private-repo", + client: privateClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "gitlab.com", + Owner: "fake-owner", + Repo: "fake-private-repo", + Branch: "", + Path: "", + token: "fake-gitlab-token", + IsFile: false, + }, + }, + { + name: "should parse private raw GitLab file path with token", + url: "https://gitlab.com/fake-owner/fake-private-repo/-/raw/main/README.md", + client: privateClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "gitlab.com", + Owner: "fake-owner", + Repo: "fake-private-repo", + Branch: "main", + Path: "README.md", + token: "fake-gitlab-token", + IsFile: true, + }, + }, + // Bitbucket + { + name: "should parse public Bitbucket repo with root path", + url: "https://bitbucket.org/fake-owner/fake-public-repo", + client: publicClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "fake-owner", + Repo: "fake-public-repo", + Branch: "", + Path: "", + token: "", + IsFile: false, + }, + }, + { + name: "should fail with only Bitbucket host", + url: "https://bitbucket.org/", + client: publicClient, + wantErr: "url path should contain /*", + }, + { + name: "should parse public Bitbucket repo with file path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/README.md", + client: publicClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "fake-owner", + Repo: "fake-public-repo", + Branch: "main", + Path: "README.md", + token: "", + IsFile: true, + }, + }, + { + name: "should parse public Bitbucket file path with nested path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/directory/test.txt", + client: publicClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "fake-owner", + Repo: "fake-public-repo", + Branch: "main", + Path: "directory/test.txt", + token: "", + IsFile: true, + }, + }, + { + name: "should parse public Bitbucket repo with raw file path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/raw/main/README.md", + client: publicClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "fake-owner", + Repo: "fake-public-repo", + Branch: "main", + Path: "README.md", + token: "", + IsFile: true, + }, + }, + { + name: "should fail with missing Bitbucket repo", + url: "https://bitbucket.org/fake-owner", + client: publicClient, + wantErr: "url path should contain /*", + }, + { + name: "should fail with invalid Bitbucket directory or file path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/main/README.md", + client: publicClient, + wantErr: "url path should contain path to directory or file*", + }, + { + name: "should fail with missing Bitbucket keywords", + url: "https://bitbucket.org/fake-owner/fake-public-repo/main/test/README.md", + client: publicClient, + wantErr: "url path should contain 'raw' or 'src'*", + }, + { + name: "should parse private Bitbucket repo with token", + url: "https://bitbucket.org/fake-owner/fake-private-repo", + client: privateClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "fake-owner", + Repo: "fake-private-repo", + Branch: "", + Path: "", + token: "fake-bitbucket-token", + IsFile: false, + }, + }, + { + name: "should parse private raw Bitbucket file path with token", + url: "https://bitbucket.org/fake-owner/fake-private-repo/raw/main/README.md", + client: privateClient, + wantUrl: GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "fake-owner", + Repo: "fake-private-repo", + Branch: "main", + Path: "README.md", + token: "fake-bitbucket-token", + IsFile: true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseGitUrlWithClient(tt.url, tt.client) + if (err != nil) != (tt.wantErr != "") { + t.Errorf("Unxpected error: %t, want: %v", err, tt.wantUrl) + } else if err == nil && !reflect.DeepEqual(got, tt.wantUrl) { + t.Errorf("Expected: %v, received: %v, difference at %v", tt.wantUrl, got, pretty.Compare(tt.wantUrl, got)) + } else if err != nil { + assert.Regexp(t, tt.wantErr, err.Error(), "Error message should match") + } + }) + } +} + +func TestCloneGitRepo(t *testing.T) { + tempDir, err := ioutil.TempDir("", "") + if err != nil { + t.Errorf("Failed to create temp dir: %s, error: %v", tempDir, err) + } + defer os.RemoveAll(tempDir) + + invalidGitUrl := GitUrl{ + Protocol: "", + Host: "", + Owner: "nonexistent", + Repo: "nonexistent", + Branch: "nonexistent", + } + + validGitHubUrl := GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Branch: "main", + } + + tests := []struct { + name string + gitUrl GitUrl + destDir string + wantErr bool + }{ + { + name: "should fail with invalid git url", + gitUrl: invalidGitUrl, + destDir: filepath.Join(os.TempDir(), "nonexistent"), + wantErr: true, + }, + { + name: "should be able to clone valid github url", + gitUrl: validGitHubUrl, + destDir: tempDir, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := CloneGitRepo(tt.gitUrl, tt.destDir) + if (err != nil) != tt.wantErr { + t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err) + } + }) + } +} diff --git a/pkg/util/util.go b/pkg/util/util.go index bdcdb4d1..f6c6fc90 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -41,11 +41,8 @@ import ( "syscall" "time" - "github.com/go-git/go-git/v5/plumbing" - "github.com/devfile/library/v2/pkg/testingutil/filesystem" "github.com/fatih/color" - gitpkg "github.com/go-git/go-git/v5" "github.com/gobwas/glob" "github.com/gregjones/httpcache" "github.com/gregjones/httpcache/diskcache" @@ -1186,57 +1183,6 @@ func ValidateFile(filePath string) error { return nil } -// GetGitUrlComponentsFromRaw converts a raw GitHub file link to a map of the url components -func GetGitUrlComponentsFromRaw(rawGitURL string) (map[string]string, error) { - var urlComponents map[string]string - - err := ValidateURL(rawGitURL) - if err != nil { - return nil, err - } - - u, _ := url.Parse(rawGitURL) - // the url scheme (e.g. https://) is removed before splitting into the 5 components - urlPath := strings.SplitN(u.Host+u.Path, "/", 5) - - // raw GitHub url: https://raw.githubusercontent.com/devfile/registry/main/stacks/nodejs/devfile.yaml - // host: raw.githubusercontent.com - // username: devfile - // project: registry - // branch: main - // file: stacks/nodejs/devfile.yaml - if len(urlPath) == 5 { - urlComponents = map[string]string{ - "host": urlPath[0], - "username": urlPath[1], - "project": urlPath[2], - "branch": urlPath[3], - "file": urlPath[4], - } - } - - return urlComponents, nil -} - -// CloneGitRepo clones a GitHub repo to a destination directory -func CloneGitRepo(gitUrlComponents map[string]string, destDir string) error { - gitUrl := fmt.Sprintf("https://github.com/%s/%s.git", gitUrlComponents["username"], gitUrlComponents["project"]) - branch := fmt.Sprintf("refs/heads/%s", gitUrlComponents["branch"]) - - cloneOptions := &gitpkg.CloneOptions{ - URL: gitUrl, - ReferenceName: plumbing.ReferenceName(branch), - SingleBranch: true, - Depth: 1, - } - - _, err := gitpkg.PlainClone(destDir, false, cloneOptions) - if err != nil { - return err - } - return nil -} - // CopyFile copies file from source path to destination path func CopyFile(srcPath string, dstPath string, info os.FileInfo) error { // In order to avoid file overriding issue, do nothing if source path is equal to destination path diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 4c0feb32..31f83656 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -1017,85 +1017,6 @@ func TestValidateFile(t *testing.T) { } } -func TestGetGitUrlComponentsFromRaw(t *testing.T) { - validRawGitUrl := "https://raw.githubusercontent.com/username/project/branch/file/path" - invalidUrl := "github.com/not/valid/url" - - tests := []struct { - name string - url string - wantErr bool - }{ - { - name: "should be able to get git url components", - url: validRawGitUrl, - wantErr: false, - }, - { - name: "should fail with invalid raw git url", - url: invalidUrl, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := GetGitUrlComponentsFromRaw(tt.url) - if (err != nil) != tt.wantErr { - t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err) - } - }) - } -} - -func TestCloneGitRepo(t *testing.T) { - tempDir, err := ioutil.TempDir("", "") - if err != nil { - t.Errorf("Failed to create temp dir: %s, error: %v", tempDir, err) - } - defer os.RemoveAll(tempDir) - - invalidGitUrl := map[string]string{ - "username": "devfile", - "project": "nonexistent", - "branch": "nonexistent", - } - validGitUrl := map[string]string{ - "username": "devfile", - "project": "library", - "branch": "main", - } - - tests := []struct { - name string - gitUrlComponents map[string]string - destDir string - wantErr bool - }{ - { - name: "should fail with invalid git url", - gitUrlComponents: invalidGitUrl, - destDir: filepath.Join(os.TempDir(), "nonexistent"), - wantErr: true, - }, - { - name: "should be able to clone valid git url", - gitUrlComponents: validGitUrl, - destDir: tempDir, - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := CloneGitRepo(tt.gitUrlComponents, tt.destDir) - if (err != nil) != tt.wantErr { - t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err) - } - }) - } -} - func TestCopyFile(t *testing.T) { // Create temp dir tempDir, err := ioutil.TempDir("", "") From 41e616169fdb794cf2952224b43037da301ada4a Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Fri, 13 Jan 2023 13:34:19 -0500 Subject: [PATCH 02/15] checking for private repo in clone function Signed-off-by: Michael Hoang --- README.md | 15 +++ go.mod | 1 - go.sum | 18 ---- pkg/devfile/parser/parse.go | 2 +- pkg/util/git.go | 127 +++++++++-------------- pkg/util/git_test.go | 196 ++++++++++++++++++------------------ 6 files changed, 163 insertions(+), 196 deletions(-) diff --git a/README.md b/README.md index 184cd0ae..0eee0d66 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,21 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g resources, err := ParseKubernetesYaml(values) ``` +8. To get resources from Git repositories + ```go + // Parse the repo url + gitUrl, err := util.ParseGitUrl(url) + + // Clone the repo to a destination dir + err = util.CloneGitRepo(gitUrl, destDir) + ``` + If repository is private, set the correct environment variables + ```shell + # credentials for private repositories + export GITHUB_TOKEN= + export GITLAB_TOKEN= + export BITBUCKET_TOKEN= + ``` ## Projects using devfile/library diff --git a/go.mod b/go.mod index 92530e38..6c5103eb 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/devfile/registry-support/registry-library v0.0.0-20221018213054-47b3ffaeadba github.com/fatih/color v1.7.0 github.com/fsnotify/fsnotify v1.4.9 - github.com/go-git/go-git/v5 v5.4.2 github.com/gobwas/glob v0.2.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.6 diff --git a/go.sum b/go.sum index 276feacd..eb341251 100644 --- a/go.sum +++ b/go.sum @@ -86,7 +86,6 @@ github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:m github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -94,7 +93,6 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -102,14 +100,12 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= @@ -342,7 +338,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -361,7 +356,6 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= @@ -372,16 +366,11 @@ github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXt github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= -github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -578,7 +567,6 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= @@ -598,7 +586,6 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -635,7 +622,6 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= @@ -657,7 +643,6 @@ github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WT github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -816,7 +801,6 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -891,7 +875,6 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= -github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1416,7 +1399,6 @@ gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76 gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index 42c17e5c..bbd3f99d 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -1,5 +1,5 @@ // -// Copyright 2022 Red Hat, Inc. +// Copyright 2022-2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/util/git.go b/pkg/util/git.go index bbf7be8d..62070285 100644 --- a/pkg/util/git.go +++ b/pkg/util/git.go @@ -1,5 +1,5 @@ // -// Copyright 2022 Red Hat, Inc. +// Copyright 2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,12 +17,9 @@ package util import ( "fmt" - gitpkg "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - githttp "github.com/go-git/go-git/v5/plumbing/transport/http" - "net/http" "net/url" "os" + "os/exec" "path/filepath" "strings" ) @@ -39,27 +36,18 @@ const ( ) type GitUrl struct { - Protocol string - Host string - Owner string - Repo string - Branch string - Path string - token string - IsFile bool + Protocol string // URL scheme + Host string // URL domain name + Owner string // name of the repo owner + Repo string // name of the repo + Branch string // branch name + Path string // path to a directory or file in the repo + token string // used for authenticating a private repo + IsFile bool // defines if the URL points to a file in the repo } // ParseGitUrl extracts information from a GitHub, GitLab, or Bitbucket url -// A client is used to check whether the url is private or public, and sets -// the providers personal access token from the environment variable func ParseGitUrl(fullUrl string) (GitUrl, error) { - var c = http.Client{ - Timeout: HTTPRequestResponseTimeout, - } - return parseGitUrlWithClient(fullUrl, c) -} - -func parseGitUrlWithClient(fullUrl string, c http.Client) (GitUrl, error) { var g GitUrl err := ValidateURL(fullUrl) @@ -77,11 +65,11 @@ func parseGitUrlWithClient(fullUrl string, c http.Client) (GitUrl, error) { } if parsedUrl.Host == RawGitHubHost || parsedUrl.Host == GitHubHost { - g, err = parseGitHubUrl(g, parsedUrl, c) + err = g.parseGitHubUrl(parsedUrl) } else if parsedUrl.Host == GitLabHost { - g, err = parseGitLabUrl(g, parsedUrl, c) + err = g.parseGitLabUrl(parsedUrl) } else if parsedUrl.Host == BitbucketHost { - g, err = parseBitbucketUrl(g, parsedUrl, c) + err = g.parseBitbucketUrl(parsedUrl) } else { err = fmt.Errorf("url host should be a valid GitHub, GitLab, or Bitbucket host; received: %s", parsedUrl.Host) } @@ -89,12 +77,13 @@ func parseGitUrlWithClient(fullUrl string, c http.Client) (GitUrl, error) { return g, err } -func parseGitHubUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { +func (g *GitUrl) parseGitHubUrl(url *url.URL) error { var splitUrl []string var err error g.Protocol = url.Scheme g.Host = url.Host + g.token = os.Getenv(GitHubToken) if g.Host == RawGitHubHost { g.IsFile = true @@ -131,20 +120,17 @@ func parseGitHubUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { } } - if !isGitUrlPublic(g, c) { - g.token = os.Getenv(GitHubToken) - } - - return g, err + return err } -func parseGitLabUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { +func (g *GitUrl) parseGitLabUrl(url *url.URL) error { var splitFile, splitOrg []string var err error g.Protocol = url.Scheme g.Host = url.Host g.IsFile = false + g.token = os.Getenv(GitLabToken) // GitLab urls contain a '-' separating the root of the repo // and the path to a file or directory @@ -175,20 +161,17 @@ func parseGitLabUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { } } - if !isGitUrlPublic(g, c) { - g.token = os.Getenv(GitLabToken) - } - - return g, err + return err } -func parseBitbucketUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { +func (g *GitUrl) parseBitbucketUrl(url *url.URL) error { var splitUrl []string var err error g.Protocol = url.Scheme g.Host = url.Host g.IsFile = false + g.token = os.Getenv(BitbucketToken) splitUrl = strings.SplitN(url.Path[1:], "/", 5) if len(splitUrl) < 2 { @@ -215,61 +198,51 @@ func parseBitbucketUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { } } - if !isGitUrlPublic(g, c) { - g.token = os.Getenv(BitbucketToken) - } - - return g, err + return err } -func isGitUrlPublic(g GitUrl, c http.Client) bool { - host := g.Host - if host == RawGitHubHost { - host = GitHubHost - } - - repo := fmt.Sprintf("%s://%s/%s/%s", g.Protocol, host, g.Owner, g.Repo) - - if res, err := c.Get(repo); err != nil { - return false - } else if res.StatusCode == http.StatusOK { - return true - } - return false -} - -// CloneGitRepo clones a GitHub Repo to a destination directory +// CloneGitRepo clones a git repo to a destination directory func CloneGitRepo(g GitUrl, destDir string) error { - var cloneOptions *gitpkg.CloneOptions - host := g.Host if host == RawGitHubHost { host = GitHubHost } + isPublic := true repoUrl := fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, host, g.Owner, g.Repo) - branch := fmt.Sprintf("refs/heads/%s", g.Branch) - cloneOptions = &gitpkg.CloneOptions{ - URL: repoUrl, - ReferenceName: plumbing.ReferenceName(branch), - SingleBranch: true, - Depth: 1, + params := HTTPRequestParams{ + URL: repoUrl, } - if g.token != "" { - cloneOptions.Auth = &githttp.BasicAuth{ - // go-git auth allows username to be anything except - // an empty string for GitHub and GitLab, however requires - // for Bitbucket to be "x-token-auth" - Username: "x-token-auth", - Password: g.token, + // check if the git repo is public + _, err := HTTPGetRequest(params, 0) + if err != nil { + // private git repo requires authentication + isPublic = false + repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) + if g.Host == BitbucketHost { + repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) } } - _, err := gitpkg.PlainClone(destDir, false, cloneOptions) + /* #nosec G204 -- user input is processed into an expected format for the git clone command */ + c := exec.Command("git", "clone", repoUrl, destDir) + c.Dir = destDir + + // set env to skip authentication prompt and directly error out + c.Env = os.Environ() + c.Env = append(c.Env, "GIT_TERMINAL_PROMPT=0", "GIT_ASKPASS=/bin/echo") + + _, err = c.CombinedOutput() if err != nil { - return err + if !isPublic { + if g.token == "" { + return fmt.Errorf("failed to clone repo without a token, ensure that a token is set if the repo is private. error: %v", err) + } + return fmt.Errorf("failed to clone repo with token, ensure that the url and token is correct. error: %v", err) + } + return fmt.Errorf("failed to clone repo, ensure that the url is correct. error: %v", err) } return nil diff --git a/pkg/util/git_test.go b/pkg/util/git_test.go index 2d62cc26..61e0e86f 100644 --- a/pkg/util/git_test.go +++ b/pkg/util/git_test.go @@ -1,5 +1,5 @@ // -// Copyright 2021-2022 Red Hat, Inc. +// Copyright 2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,8 +18,6 @@ package util import ( "github.com/kylelemons/godebug/pretty" "github.com/stretchr/testify/assert" - "io/ioutil" - "net/http" "os" "path/filepath" "reflect" @@ -32,22 +30,7 @@ var ( bitbucketToken = "fake-bitbucket-token" ) -type respondWithStatus struct { - status int -} - -func (rs respondWithStatus) RoundTrip(*http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: rs.status, - }, nil -} - -var ( - publicClient = http.Client{Transport: respondWithStatus{status: http.StatusOK}} - privateClient = http.Client{Transport: respondWithStatus{status: http.StatusNotFound}} -) - -func Test_parseGitUrlWithClient(t *testing.T) { +func Test_ParseGitUrl(t *testing.T) { defer func() { err := os.Unsetenv(githubToken) if err != nil { @@ -79,27 +62,23 @@ func Test_parseGitUrlWithClient(t *testing.T) { tests := []struct { name string url string - client http.Client wantUrl GitUrl wantErr string }{ { name: "should fail with empty url", url: "", - client: publicClient, wantErr: "URL is invalid", }, { name: "should fail with invalid git host", url: "https://google.ca/", - client: publicClient, wantErr: "url host should be a valid GitHub, GitLab, or Bitbucket host*", }, // GitHub { - name: "should parse public GitHub repo with root path", - url: "https://github.com/devfile/library", - client: publicClient, + name: "should parse public GitHub repo with root path", + url: "https://github.com/devfile/library", wantUrl: GitUrl{ Protocol: "https", Host: "github.com", @@ -107,20 +86,18 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "library", Branch: "", Path: "", - token: "", + token: "fake-github-token", IsFile: false, }, }, { name: "should fail with only GitHub host", url: "https://github.com/", - client: publicClient, wantErr: "url path should contain /*", }, { - name: "should parse public GitHub repo with file path", - url: "https://github.com/devfile/library/blob/main/devfile.yaml", - client: publicClient, + name: "should parse public GitHub repo with file path", + url: "https://github.com/devfile/library/blob/main/devfile.yaml", wantUrl: GitUrl{ Protocol: "https", Host: "github.com", @@ -128,14 +105,13 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "library", Branch: "main", Path: "devfile.yaml", - token: "", + token: "fake-github-token", IsFile: true, }, }, { - name: "should parse public GitHub repo with raw file path", - url: "https://raw.githubusercontent.com/devfile/library/main/devfile.yaml", - client: publicClient, + name: "should parse public GitHub repo with raw file path", + url: "https://raw.githubusercontent.com/devfile/library/main/devfile.yaml", wantUrl: GitUrl{ Protocol: "https", Host: "raw.githubusercontent.com", @@ -143,26 +119,23 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "library", Branch: "main", Path: "devfile.yaml", - token: "", + token: "fake-github-token", IsFile: true, }, }, { name: "should fail with missing GitHub repo", url: "https://github.com/devfile", - client: publicClient, wantErr: "url path should contain /*", }, { name: "should fail with invalid GitHub raw file path", url: "https://raw.githubusercontent.com/devfile/library/devfile.yaml", - client: publicClient, wantErr: "raw url path should contain ///*", }, { - name: "should parse private GitHub repo with token", - url: "https://github.com/fake-owner/fake-private-repo", - client: privateClient, + name: "should parse private GitHub repo with token", + url: "https://github.com/fake-owner/fake-private-repo", wantUrl: GitUrl{ Protocol: "https", Host: "github.com", @@ -175,9 +148,8 @@ func Test_parseGitUrlWithClient(t *testing.T) { }, }, { - name: "should parse private raw GitHub file path with token", - url: "https://raw.githubusercontent.com/fake-owner/fake-private-repo/main/README.md", - client: privateClient, + name: "should parse private raw GitHub file path with token", + url: "https://raw.githubusercontent.com/fake-owner/fake-private-repo/main/README.md", wantUrl: GitUrl{ Protocol: "https", Host: "raw.githubusercontent.com", @@ -191,9 +163,8 @@ func Test_parseGitUrlWithClient(t *testing.T) { }, // Gitlab { - name: "should parse public GitLab repo with root path", - url: "https://gitlab.com/gitlab-org/gitlab-foss", - client: publicClient, + name: "should parse public GitLab repo with root path", + url: "https://gitlab.com/gitlab-org/gitlab-foss", wantUrl: GitUrl{ Protocol: "https", Host: "gitlab.com", @@ -201,20 +172,18 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "gitlab-foss", Branch: "", Path: "", - token: "", + token: "fake-gitlab-token", IsFile: false, }, }, { name: "should fail with only GitLab host", url: "https://gitlab.com/", - client: publicClient, wantErr: "url path should contain /*", }, { - name: "should parse public GitLab repo with file path", - url: "https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/README.md", - client: publicClient, + name: "should parse public GitLab repo with file path", + url: "https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/README.md", wantUrl: GitUrl{ Protocol: "https", Host: "gitlab.com", @@ -222,26 +191,23 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "gitlab-foss", Branch: "master", Path: "README.md", - token: "", + token: "fake-gitlab-token", IsFile: true, }, }, { name: "should fail with missing GitLab repo", url: "https://gitlab.com/gitlab-org", - client: publicClient, wantErr: "url path should contain /*", }, { name: "should fail with missing GitLab keywords", url: "https://gitlab.com/gitlab-org/gitlab-foss/-/master/directory/README.md", - client: publicClient, wantErr: "url path should contain 'blob' or 'tree' or 'raw'*", }, { - name: "should parse private GitLab repo with token", - url: "https://gitlab.com/fake-owner/fake-private-repo", - client: privateClient, + name: "should parse private GitLab repo with token", + url: "https://gitlab.com/fake-owner/fake-private-repo", wantUrl: GitUrl{ Protocol: "https", Host: "gitlab.com", @@ -254,9 +220,8 @@ func Test_parseGitUrlWithClient(t *testing.T) { }, }, { - name: "should parse private raw GitLab file path with token", - url: "https://gitlab.com/fake-owner/fake-private-repo/-/raw/main/README.md", - client: privateClient, + name: "should parse private raw GitLab file path with token", + url: "https://gitlab.com/fake-owner/fake-private-repo/-/raw/main/README.md", wantUrl: GitUrl{ Protocol: "https", Host: "gitlab.com", @@ -270,9 +235,8 @@ func Test_parseGitUrlWithClient(t *testing.T) { }, // Bitbucket { - name: "should parse public Bitbucket repo with root path", - url: "https://bitbucket.org/fake-owner/fake-public-repo", - client: publicClient, + name: "should parse public Bitbucket repo with root path", + url: "https://bitbucket.org/fake-owner/fake-public-repo", wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", @@ -280,20 +244,18 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "fake-public-repo", Branch: "", Path: "", - token: "", + token: "fake-bitbucket-token", IsFile: false, }, }, { name: "should fail with only Bitbucket host", url: "https://bitbucket.org/", - client: publicClient, wantErr: "url path should contain /*", }, { - name: "should parse public Bitbucket repo with file path", - url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/README.md", - client: publicClient, + name: "should parse public Bitbucket repo with file path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/README.md", wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", @@ -301,14 +263,13 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "fake-public-repo", Branch: "main", Path: "README.md", - token: "", + token: "fake-bitbucket-token", IsFile: true, }, }, { - name: "should parse public Bitbucket file path with nested path", - url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/directory/test.txt", - client: publicClient, + name: "should parse public Bitbucket file path with nested path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/directory/test.txt", wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", @@ -316,14 +277,13 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "fake-public-repo", Branch: "main", Path: "directory/test.txt", - token: "", + token: "fake-bitbucket-token", IsFile: true, }, }, { - name: "should parse public Bitbucket repo with raw file path", - url: "https://bitbucket.org/fake-owner/fake-public-repo/raw/main/README.md", - client: publicClient, + name: "should parse public Bitbucket repo with raw file path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/raw/main/README.md", wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", @@ -331,32 +291,28 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "fake-public-repo", Branch: "main", Path: "README.md", - token: "", + token: "fake-bitbucket-token", IsFile: true, }, }, { name: "should fail with missing Bitbucket repo", url: "https://bitbucket.org/fake-owner", - client: publicClient, wantErr: "url path should contain /*", }, { name: "should fail with invalid Bitbucket directory or file path", url: "https://bitbucket.org/fake-owner/fake-public-repo/main/README.md", - client: publicClient, wantErr: "url path should contain path to directory or file*", }, { name: "should fail with missing Bitbucket keywords", url: "https://bitbucket.org/fake-owner/fake-public-repo/main/test/README.md", - client: publicClient, wantErr: "url path should contain 'raw' or 'src'*", }, { - name: "should parse private Bitbucket repo with token", - url: "https://bitbucket.org/fake-owner/fake-private-repo", - client: privateClient, + name: "should parse private Bitbucket repo with token", + url: "https://bitbucket.org/fake-owner/fake-private-repo", wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", @@ -369,9 +325,8 @@ func Test_parseGitUrlWithClient(t *testing.T) { }, }, { - name: "should parse private raw Bitbucket file path with token", - url: "https://bitbucket.org/fake-owner/fake-private-repo/raw/main/README.md", - client: privateClient, + name: "should parse private raw Bitbucket file path with token", + url: "https://bitbucket.org/fake-owner/fake-private-repo/raw/main/README.md", wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", @@ -387,7 +342,7 @@ func Test_parseGitUrlWithClient(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseGitUrlWithClient(tt.url, tt.client) + got, err := ParseGitUrl(tt.url) if (err != nil) != (tt.wantErr != "") { t.Errorf("Unxpected error: %t, want: %v", err, tt.wantUrl) } else if err == nil && !reflect.DeepEqual(got, tt.wantUrl) { @@ -400,11 +355,9 @@ func Test_parseGitUrlWithClient(t *testing.T) { } func TestCloneGitRepo(t *testing.T) { - tempDir, err := ioutil.TempDir("", "") - if err != nil { - t.Errorf("Failed to create temp dir: %s, error: %v", tempDir, err) - } - defer os.RemoveAll(tempDir) + tempDirGitHub := t.TempDir() + tempDirGitLab := t.TempDir() + tempDirBitbucket := t.TempDir() invalidGitUrl := GitUrl{ Protocol: "", @@ -422,31 +375,76 @@ func TestCloneGitRepo(t *testing.T) { Branch: "main", } + validGitLabUrl := GitUrl{ + Protocol: "https", + Host: "gitlab.com", + Owner: "mike-hoang", + Repo: "public-testing-repo", + Branch: "main", + } + + validBitbucketUrl := GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "mike-hoang", + Repo: "public-testing-repo", + Branch: "master", + } + + privateGitHubRepo := GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "fake-owner", + Repo: "fake-private-repo", + Branch: "master", + token: "fake-github-token", + } + + privateRepoMissingTokenErr := "failed to clone repo without a token*" + privateRepoBadTokenErr := "failed to clone repo with token*" + tests := []struct { name string gitUrl GitUrl destDir string - wantErr bool + wantErr string }{ { name: "should fail with invalid git url", gitUrl: invalidGitUrl, destDir: filepath.Join(os.TempDir(), "nonexistent"), - wantErr: true, + wantErr: privateRepoMissingTokenErr, }, { - name: "should be able to clone valid github url", + name: "should fail to clone valid private git url with a bad token", + gitUrl: privateGitHubRepo, + destDir: filepath.Join(os.TempDir(), "nonexistent"), + wantErr: privateRepoBadTokenErr, + }, + { + name: "should be able to clone valid public github url", gitUrl: validGitHubUrl, - destDir: tempDir, - wantErr: false, + destDir: tempDirGitHub, + }, + { + name: "should be able to clone valid public gitlab url", + gitUrl: validGitLabUrl, + destDir: tempDirGitLab, + }, + { + name: "should be able to clone valid public bitbucket url", + gitUrl: validBitbucketUrl, + destDir: tempDirBitbucket, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := CloneGitRepo(tt.gitUrl, tt.destDir) - if (err != nil) != tt.wantErr { - t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err) + if (err != nil) != (tt.wantErr != "") { + t.Errorf("Unxpected error: %t, want: %v", err, tt.wantErr) + } else if err != nil { + assert.Regexp(t, tt.wantErr, err.Error(), "Error message should match") } }) } From 60fa9f5586dc24717a20beb417491c5a03a5612f Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Tue, 17 Jan 2023 14:44:07 -0500 Subject: [PATCH 03/15] adding private repo check before clone attempt Signed-off-by: Michael Hoang --- README.md | 31 +++++++++++---- pkg/devfile/parser/parse.go | 9 +++-- pkg/devfile/parser/parse_test.go | 4 +- pkg/util/git.go | 68 ++++++++++++++++++++++++-------- pkg/util/git_test.go | 40 ++++++++++++------- 5 files changed, 108 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 0eee0d66..a4c39a8f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,25 @@ The Devfile Parser library is a Golang module that: 3. generates Kubernetes objects for the various devfile resources. 4. defines util functions for the devfile. +## Private Repository Support + +Tokens are required to be set in the following cases: +1. tooling client calling the library API +2. parsing a devfile from a private repository +3. parsing a devfile containing a parent devfile from a private repository + +Set the environment variables for the necessary git providers: + ```shell + export GITHUB_TOKEN= + export GITLAB_TOKEN= + export BITBUCKET_TOKEN= + ``` + +For more information about personal access tokens: +1. [GitHub docs](https://docs.github.com/en/enterprise-server@3.4/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) +2. [GitLab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token) +3. [Bitbucket docs](https://support.atlassian.com/bitbucket-cloud/docs/repository-access-tokens/) + ## Usage The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/github.com/devfile/library). @@ -169,19 +188,17 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g 8. To get resources from Git repositories ```go + // Supported url formats + url = "https://github.com//" + url = "https://bitbucket.org//" + url = "https://gitlab.com//" + // Parse the repo url gitUrl, err := util.ParseGitUrl(url) // Clone the repo to a destination dir err = util.CloneGitRepo(gitUrl, destDir) ``` - If repository is private, set the correct environment variables - ```shell - # credentials for private repositories - export GITHUB_TOKEN= - export GITLAB_TOKEN= - export BITBUCKET_TOKEN= - ``` ## Projects using devfile/library diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index bbd3f99d..556a85a0 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -425,14 +425,15 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D } d.Ctx = devfileCtx.NewURLDevfileCtx(newUri) - if strings.Contains(newUri, util.RawGitHubHost) { + if strings.Contains(newUri, util.RawGitHubHost) || strings.Contains(newUri, util.GitHubHost) || + strings.Contains(newUri, util.GitLabHost) || strings.Contains(newUri, util.BitbucketToken) { gitUrl, err := util.ParseGitUrl(newUri) if err != nil { return DevfileObj{}, err } if gitUrl.IsFile { destDir := path.Dir(curDevfileCtx.GetAbsPath()) - err = getResourcesFromGit(gitUrl, destDir) + err = getResourcesFromGit(gitUrl, destDir, tool.httpTimeout) if err != nil { return DevfileObj{}, err } @@ -445,14 +446,14 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D return populateAndParseDevfile(d, newResolveCtx, tool, true) } -func getResourcesFromGit(g util.GitUrl, destDir string) error { +func getResourcesFromGit(g util.GitUrl, destDir string, httpTimeout *int) error { stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) if err != nil { return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) } defer os.RemoveAll(stackDir) - err = util.CloneGitRepo(g, stackDir) + err = util.CloneGitRepo(g, stackDir, httpTimeout) if err != nil { return err } diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index 50223bad..386cf703 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -4166,6 +4166,8 @@ func Test_getResourcesFromGit(t *testing.T) { } defer os.RemoveAll(destDir) + httpTimeout := 0 + invalidGitHubUrl := util.GitUrl{ Protocol: "", Host: "", @@ -4205,7 +4207,7 @@ func Test_getResourcesFromGit(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := getResourcesFromGit(tt.gitUrl, tt.destDir) + err := getResourcesFromGit(tt.gitUrl, tt.destDir, &httpTimeout) if (err != nil) != tt.wantErr { t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err) } diff --git a/pkg/util/git.go b/pkg/util/git.go index 62070285..829ec62b 100644 --- a/pkg/util/git.go +++ b/pkg/util/git.go @@ -201,28 +201,68 @@ func (g *GitUrl) parseBitbucketUrl(url *url.URL) error { return err } +// ValidateToken makes a http get request to the repo with the GitUrl token +// Returns an error if the get request fails +// If token is empty or invalid and validate succeeds, the repository is public +func (g *GitUrl) ValidateToken(params HTTPRequestParams) error { + var apiUrl string + + switch g.Host { + case GitHubHost, RawGitHubHost: + apiUrl = fmt.Sprintf("https://api.github.com/repos/%s/%s", g.Owner, g.Repo) + case GitLabHost: + apiUrl = fmt.Sprintf("https://gitlab.com/api/v4/projects/%s%%2F%s", g.Owner, g.Repo) + case BitbucketHost: + apiUrl = fmt.Sprintf("https://api.bitbucket.org/2.0/repositories/%s/%s", g.Owner, g.Repo) + default: + apiUrl = fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, g.Host, g.Owner, g.Repo) + } + + params.URL = apiUrl + res, err := HTTPGetRequest(params, 0) + if len(res) == 0 || err != nil { + return err + } + + return nil +} + // CloneGitRepo clones a git repo to a destination directory -func CloneGitRepo(g GitUrl, destDir string) error { +// Only supports git repositories hosted on GitHub, GitLab, and Bitbucket +func CloneGitRepo(g GitUrl, destDir string, httpTimeout *int) error { + exist := CheckPathExists(destDir) + if !exist { + return fmt.Errorf("failed to clone repo, destination directory: '%s' does not exists", destDir) + } + host := g.Host if host == RawGitHubHost { host = GitHubHost } - isPublic := true repoUrl := fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, host, g.Owner, g.Repo) params := HTTPRequestParams{ - URL: repoUrl, + Timeout: httpTimeout, } - // check if the git repo is public - _, err := HTTPGetRequest(params, 0) + // public repos will succeed even if token is invalid or empty + err := g.ValidateToken(params) + if err != nil { - // private git repo requires authentication - isPublic = false - repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) - if g.Host == BitbucketHost { - repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) + if g.token != "" { + params.Token = g.token + err := g.ValidateToken(params) + if err != nil { + return fmt.Errorf("failed to validate git url with token, ensure that the url and token is correct. error: %v", err) + } else { + repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) + if g.Host == BitbucketHost { + repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) + } + } + } else { + return fmt.Errorf("failed to validate git url without a token, ensure that a token is set if the repo is private. error: %v", err) } } @@ -236,13 +276,7 @@ func CloneGitRepo(g GitUrl, destDir string) error { _, err = c.CombinedOutput() if err != nil { - if !isPublic { - if g.token == "" { - return fmt.Errorf("failed to clone repo without a token, ensure that a token is set if the repo is private. error: %v", err) - } - return fmt.Errorf("failed to clone repo with token, ensure that the url and token is correct. error: %v", err) - } - return fmt.Errorf("failed to clone repo, ensure that the url is correct. error: %v", err) + return fmt.Errorf("failed to clone repo, ensure that the git url is correct. error: %v", err) } return nil diff --git a/pkg/util/git_test.go b/pkg/util/git_test.go index 61e0e86f..52cb33f2 100644 --- a/pkg/util/git_test.go +++ b/pkg/util/git_test.go @@ -355,10 +355,13 @@ func Test_ParseGitUrl(t *testing.T) { } func TestCloneGitRepo(t *testing.T) { + tempInvalidDir := t.TempDir() tempDirGitHub := t.TempDir() tempDirGitLab := t.TempDir() tempDirBitbucket := t.TempDir() + httpTimeout := 0 + invalidGitUrl := GitUrl{ Protocol: "", Host: "", @@ -367,7 +370,7 @@ func TestCloneGitRepo(t *testing.T) { Branch: "nonexistent", } - validGitHubUrl := GitUrl{ + validPublicGitHubUrl := GitUrl{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -375,7 +378,7 @@ func TestCloneGitRepo(t *testing.T) { Branch: "main", } - validGitLabUrl := GitUrl{ + validPublicGitLabUrl := GitUrl{ Protocol: "https", Host: "gitlab.com", Owner: "mike-hoang", @@ -383,7 +386,7 @@ func TestCloneGitRepo(t *testing.T) { Branch: "main", } - validBitbucketUrl := GitUrl{ + validPublicBitbucketUrl := GitUrl{ Protocol: "https", Host: "bitbucket.org", Owner: "mike-hoang", @@ -391,7 +394,7 @@ func TestCloneGitRepo(t *testing.T) { Branch: "master", } - privateGitHubRepo := GitUrl{ + invalidPrivateGitHubRepo := GitUrl{ Protocol: "https", Host: "github.com", Owner: "fake-owner", @@ -400,8 +403,9 @@ func TestCloneGitRepo(t *testing.T) { token: "fake-github-token", } - privateRepoMissingTokenErr := "failed to clone repo without a token*" - privateRepoBadTokenErr := "failed to clone repo with token*" + privateRepoBadTokenErr := "failed to validate git url with token*" + publicRepoInvalidUrlErr := "failed to validate git url without a token" + missingDestDirErr := "failed to clone repo, destination directory*" tests := []struct { name string @@ -410,37 +414,43 @@ func TestCloneGitRepo(t *testing.T) { wantErr string }{ { - name: "should fail with invalid git url", + name: "should fail with invalid destination directory", gitUrl: invalidGitUrl, destDir: filepath.Join(os.TempDir(), "nonexistent"), - wantErr: privateRepoMissingTokenErr, + wantErr: missingDestDirErr, }, { - name: "should fail to clone valid private git url with a bad token", - gitUrl: privateGitHubRepo, - destDir: filepath.Join(os.TempDir(), "nonexistent"), + name: "should fail with invalid git url", + gitUrl: invalidGitUrl, + destDir: tempInvalidDir, + wantErr: publicRepoInvalidUrlErr, + }, + { + name: "should fail to clone invalid private git url with a bad token", + gitUrl: invalidPrivateGitHubRepo, + destDir: tempInvalidDir, wantErr: privateRepoBadTokenErr, }, { name: "should be able to clone valid public github url", - gitUrl: validGitHubUrl, + gitUrl: validPublicGitHubUrl, destDir: tempDirGitHub, }, { name: "should be able to clone valid public gitlab url", - gitUrl: validGitLabUrl, + gitUrl: validPublicGitLabUrl, destDir: tempDirGitLab, }, { name: "should be able to clone valid public bitbucket url", - gitUrl: validBitbucketUrl, + gitUrl: validPublicBitbucketUrl, destDir: tempDirBitbucket, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := CloneGitRepo(tt.gitUrl, tt.destDir) + err := CloneGitRepo(tt.gitUrl, tt.destDir, &httpTimeout) if (err != nil) != (tt.wantErr != "") { t.Errorf("Unxpected error: %t, want: %v", err, tt.wantErr) } else if err != nil { From 80cadccd06c98b72598f0c7df8b914500b5f870c Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Fri, 10 Mar 2023 09:41:54 -0500 Subject: [PATCH 04/15] wip: set token outside of cloning function Signed-off-by: Michael Hoang --- README.md | 20 +++-- pkg/devfile/parser/parse.go | 21 +++-- pkg/devfile/parser/parse_test.go | 2 +- pkg/util/git.go | 89 +++++++++++--------- pkg/util/git_test.go | 135 ++++++++++++++++++++----------- pkg/util/util.go | 15 ++++ pkg/util/util_test.go | 5 ++ 7 files changed, 185 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index a4c39a8f..d62a1a43 100644 --- a/README.md +++ b/README.md @@ -18,20 +18,24 @@ The Devfile Parser library is a Golang module that: Tokens are required to be set in the following cases: 1. tooling client calling the library API 2. parsing a devfile from a private repository -3. parsing a devfile containing a parent devfile from a private repository - -Set the environment variables for the necessary git providers: - ```shell - export GITHUB_TOKEN= - export GITLAB_TOKEN= - export BITBUCKET_TOKEN= - ``` +3. parsing a devfile containing a parent devfile from a private repository [1] + +Set the token for the repository: +```go +parser.ParserArgs{ + ... + Token: + ... +} +``` For more information about personal access tokens: 1. [GitHub docs](https://docs.github.com/en/enterprise-server@3.4/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) 2. [GitLab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token) 3. [Bitbucket docs](https://support.atlassian.com/bitbucket-cloud/docs/repository-access-tokens/) +[1] currently, this works under the assumption that the token can authenticate the devfile and the parent devfile; both devfiles are in the same repository. + ## Usage The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/github.com/devfile/library). diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index 556a85a0..0d3672e9 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -97,6 +97,8 @@ type ParserArgs struct { // RegistryURLs is a list of registry hosts which parser should pull parent devfile from. // If registryUrl is defined in devfile, this list will be ignored. RegistryURLs []string + // Token is a GitHub, GitLab, or Bitbucket personal access token used with a private git repo uri + Token string // DefaultNamespace is the default namespace to use // If namespace is defined under devfile's parent kubernetes object, this namespace will be ignored. DefaultNamespace string @@ -132,6 +134,7 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) { context: args.Context, k8sClient: args.K8sClient, httpTimeout: args.HTTPTimeout, + token: args.Token, } flattenedDevfile := true @@ -175,6 +178,8 @@ type resolverTools struct { // RegistryURLs is a list of registry hosts which parser should pull parent devfile from. // If registryUrl is defined in devfile, this list will be ignored. registryURLs []string + // Token is a GitHub, GitLab, or Bitbucket personal access token used with a private git repo uri + token string // Context is the context used for making Kubernetes or HTTP requests context context.Context // K8sClient is the Kubernetes client instance used for interacting with a cluster @@ -425,15 +430,14 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D } d.Ctx = devfileCtx.NewURLDevfileCtx(newUri) - if strings.Contains(newUri, util.RawGitHubHost) || strings.Contains(newUri, util.GitHubHost) || - strings.Contains(newUri, util.GitLabHost) || strings.Contains(newUri, util.BitbucketToken) { + if util.IsGitProviderRepo(newUri) { gitUrl, err := util.ParseGitUrl(newUri) if err != nil { return DevfileObj{}, err } if gitUrl.IsFile { destDir := path.Dir(curDevfileCtx.GetAbsPath()) - err = getResourcesFromGit(gitUrl, destDir, tool.httpTimeout) + err = getResourcesFromGit(gitUrl, destDir, tool.httpTimeout, tool.token) if err != nil { return DevfileObj{}, err } @@ -446,14 +450,21 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D return populateAndParseDevfile(d, newResolveCtx, tool, true) } -func getResourcesFromGit(g util.GitUrl, destDir string, httpTimeout *int) error { +func getResourcesFromGit(g util.GitUrl, destDir string, httpTimeout *int, repoToken string) error { stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) if err != nil { return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) } defer os.RemoveAll(stackDir) - err = util.CloneGitRepo(g, stackDir, httpTimeout) + if !g.IsPublic(httpTimeout) { + err = g.SetToken(repoToken, httpTimeout) + if err != nil { + return err + } + } + + err = util.CloneGitRepo(g, stackDir) if err != nil { return err } diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index 386cf703..e250192e 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -4207,7 +4207,7 @@ func Test_getResourcesFromGit(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := getResourcesFromGit(tt.gitUrl, tt.destDir, &httpTimeout) + err := getResourcesFromGit(tt.gitUrl, tt.destDir, &httpTimeout, "") if (err != nil) != tt.wantErr { t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err) } diff --git a/pkg/util/git.go b/pkg/util/git.go index 829ec62b..ce548d0b 100644 --- a/pkg/util/git.go +++ b/pkg/util/git.go @@ -29,10 +29,6 @@ const ( RawGitHubHost string = "raw.githubusercontent.com" GitLabHost string = "gitlab.com" BitbucketHost string = "bitbucket.org" - - GitHubToken string = "GITHUB_TOKEN" - GitLabToken string = "GITLAB_TOKEN" - BitbucketToken string = "BITBUCKET_TOKEN" ) type GitUrl struct { @@ -46,7 +42,8 @@ type GitUrl struct { IsFile bool // defines if the URL points to a file in the repo } -// ParseGitUrl extracts information from a GitHub, GitLab, or Bitbucket url +// ParseGitUrl extracts information from a support git url +// Only supports git repositories hosted on GitHub, GitLab, and Bitbucket func ParseGitUrl(fullUrl string) (GitUrl, error) { var g GitUrl @@ -83,7 +80,6 @@ func (g *GitUrl) parseGitHubUrl(url *url.URL) error { g.Protocol = url.Scheme g.Host = url.Host - g.token = os.Getenv(GitHubToken) if g.Host == RawGitHubHost { g.IsFile = true @@ -130,7 +126,6 @@ func (g *GitUrl) parseGitLabUrl(url *url.URL) error { g.Protocol = url.Scheme g.Host = url.Host g.IsFile = false - g.token = os.Getenv(GitLabToken) // GitLab urls contain a '-' separating the root of the repo // and the path to a file or directory @@ -171,7 +166,6 @@ func (g *GitUrl) parseBitbucketUrl(url *url.URL) error { g.Protocol = url.Scheme g.Host = url.Host g.IsFile = false - g.token = os.Getenv(BitbucketToken) splitUrl = strings.SplitN(url.Path[1:], "/", 5) if len(splitUrl) < 2 { @@ -201,10 +195,31 @@ func (g *GitUrl) parseBitbucketUrl(url *url.URL) error { return err } -// ValidateToken makes a http get request to the repo with the GitUrl token +// SetToken validates the token with a get request to the repo before setting the token +// Defaults token to empty on failure. +func (g *GitUrl) SetToken(token string, httpTimeout *int) error { + err := g.validateToken(HTTPRequestParams{Token: token, Timeout: httpTimeout}) + if err != nil { + g.token = "" + return fmt.Errorf("failed to set token. error: %v", err) + } + g.token = token + return nil +} + +// IsPublic checks if the GitUrl is public with a get request to the repo using an empty token +// Returns true if the request succeeds +func (g *GitUrl) IsPublic(httpTimeout *int) bool { + err := g.validateToken(HTTPRequestParams{Token: "", Timeout: httpTimeout}) + if err != nil { + return false + } + return true +} + +// validateToken makes a http get request to the repo with the GitUrl token // Returns an error if the get request fails -// If token is empty or invalid and validate succeeds, the repository is public -func (g *GitUrl) ValidateToken(params HTTPRequestParams) error { +func (g *GitUrl) validateToken(params HTTPRequestParams) error { var apiUrl string switch g.Host { @@ -227,9 +242,17 @@ func (g *GitUrl) ValidateToken(params HTTPRequestParams) error { return nil } -// CloneGitRepo clones a git repo to a destination directory -// Only supports git repositories hosted on GitHub, GitLab, and Bitbucket -func CloneGitRepo(g GitUrl, destDir string, httpTimeout *int) error { +// IsGitProviderRepo checks if the url matches a repo from a supported git provider +func IsGitProviderRepo(url string) bool { + if strings.Contains(url, RawGitHubHost) || strings.Contains(url, GitHubHost) || + strings.Contains(url, GitLabHost) || strings.Contains(url, BitbucketHost) { + return true + } + return false +} + +// CloneGitRepo clones a git repo to a destination directory (either an absolute or relative path) +func CloneGitRepo(g GitUrl, destDir string) error { exist := CheckPathExists(destDir) if !exist { return fmt.Errorf("failed to clone repo, destination directory: '%s' does not exists", destDir) @@ -240,29 +263,13 @@ func CloneGitRepo(g GitUrl, destDir string, httpTimeout *int) error { host = GitHubHost } - repoUrl := fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, host, g.Owner, g.Repo) - - params := HTTPRequestParams{ - Timeout: httpTimeout, - } - - // public repos will succeed even if token is invalid or empty - err := g.ValidateToken(params) - - if err != nil { - if g.token != "" { - params.Token = g.token - err := g.ValidateToken(params) - if err != nil { - return fmt.Errorf("failed to validate git url with token, ensure that the url and token is correct. error: %v", err) - } else { - repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) - if g.Host == BitbucketHost { - repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) - } - } - } else { - return fmt.Errorf("failed to validate git url without a token, ensure that a token is set if the repo is private. error: %v", err) + var repoUrl string + if g.token == "" { + repoUrl = fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, host, g.Owner, g.Repo) + } else { + repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) + if g.Host == BitbucketHost { + repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) } } @@ -274,9 +281,13 @@ func CloneGitRepo(g GitUrl, destDir string, httpTimeout *int) error { c.Env = os.Environ() c.Env = append(c.Env, "GIT_TERMINAL_PROMPT=0", "GIT_ASKPASS=/bin/echo") - _, err = c.CombinedOutput() + _, err := c.CombinedOutput() if err != nil { - return fmt.Errorf("failed to clone repo, ensure that the git url is correct. error: %v", err) + if g.token == "" { + return fmt.Errorf("failed to clone repo without a token, ensure that a token is set if the repo is private. error: %v", err) + } else { + return fmt.Errorf("failed to clone repo with token, ensure that the url and token is correct. error: %v", err) + } } return nil diff --git a/pkg/util/git_test.go b/pkg/util/git_test.go index 52cb33f2..86435e00 100644 --- a/pkg/util/git_test.go +++ b/pkg/util/git_test.go @@ -31,34 +31,6 @@ var ( ) func Test_ParseGitUrl(t *testing.T) { - defer func() { - err := os.Unsetenv(githubToken) - if err != nil { - t.Errorf("Failed to unset GitHub token") - } - err = os.Unsetenv(gitlabToken) - if err != nil { - t.Errorf("Failed to unset GitLab token") - } - err = os.Unsetenv(bitbucketToken) - if err != nil { - t.Errorf("Failed to unset Bitbucket token") - } - }() - - err := os.Setenv("GITHUB_TOKEN", githubToken) - if err != nil { - t.Errorf("Failed to set GitHub token") - } - err = os.Setenv("GITLAB_TOKEN", gitlabToken) - if err != nil { - t.Errorf("Failed to set GitLab token") - } - err = os.Setenv("BITBUCKET_TOKEN", bitbucketToken) - if err != nil { - t.Errorf("Failed to set Bitbucket token") - } - tests := []struct { name string url string @@ -86,7 +58,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "library", Branch: "", Path: "", - token: "fake-github-token", + token: "", IsFile: false, }, }, @@ -105,7 +77,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "library", Branch: "main", Path: "devfile.yaml", - token: "fake-github-token", + token: "", IsFile: true, }, }, @@ -119,7 +91,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "library", Branch: "main", Path: "devfile.yaml", - token: "fake-github-token", + token: "", IsFile: true, }, }, @@ -143,7 +115,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "fake-private-repo", Branch: "", Path: "", - token: "fake-github-token", + token: "", IsFile: false, }, }, @@ -157,7 +129,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "fake-private-repo", Branch: "main", Path: "README.md", - token: "fake-github-token", + token: "", IsFile: true, }, }, @@ -172,7 +144,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "gitlab-foss", Branch: "", Path: "", - token: "fake-gitlab-token", + token: "", IsFile: false, }, }, @@ -191,7 +163,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "gitlab-foss", Branch: "master", Path: "README.md", - token: "fake-gitlab-token", + token: "", IsFile: true, }, }, @@ -215,7 +187,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "fake-private-repo", Branch: "", Path: "", - token: "fake-gitlab-token", + token: "", IsFile: false, }, }, @@ -229,7 +201,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "fake-private-repo", Branch: "main", Path: "README.md", - token: "fake-gitlab-token", + token: "", IsFile: true, }, }, @@ -244,7 +216,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "fake-public-repo", Branch: "", Path: "", - token: "fake-bitbucket-token", + token: "", IsFile: false, }, }, @@ -263,7 +235,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "fake-public-repo", Branch: "main", Path: "README.md", - token: "fake-bitbucket-token", + token: "", IsFile: true, }, }, @@ -277,7 +249,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "fake-public-repo", Branch: "main", Path: "directory/test.txt", - token: "fake-bitbucket-token", + token: "", IsFile: true, }, }, @@ -291,7 +263,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "fake-public-repo", Branch: "main", Path: "README.md", - token: "fake-bitbucket-token", + token: "", IsFile: true, }, }, @@ -320,7 +292,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "fake-private-repo", Branch: "", Path: "", - token: "fake-bitbucket-token", + token: "", IsFile: false, }, }, @@ -334,7 +306,7 @@ func Test_ParseGitUrl(t *testing.T) { Repo: "fake-private-repo", Branch: "main", Path: "README.md", - token: "fake-bitbucket-token", + token: "", IsFile: true, }, }, @@ -354,14 +326,79 @@ func Test_ParseGitUrl(t *testing.T) { } } -func TestCloneGitRepo(t *testing.T) { +// todo: try mocking +func Test_SetToken(t *testing.T) { + g := GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Branch: "main", + token: "", + } + + httpTimeout := 0 + token := "fake-git-token" + + err := g.SetToken(token, &httpTimeout) + assert.NoError(t, err) + assert.Equal(t, token, g.token) +} + +func Test_IsPublic(t *testing.T) { + publicGitUrl := GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Branch: "main", + token: "fake-token", + } + + privateGitUrl := GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "not", + Repo: "a-valid", + Branch: "none", + token: "fake-token", + } + + httpTimeout := 0 + + tests := []struct { + name string + g GitUrl + want bool + }{ + { + name: "should be public", + g: publicGitUrl, + want: true, + }, + { + name: "should be private", + g: privateGitUrl, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.g.IsPublic(&httpTimeout) + if !reflect.DeepEqual(result, tt.want) { + t.Errorf("Got: %t, want: %t", result, tt.want) + } + }) + } +} + +func Test_CloneGitRepo(t *testing.T) { tempInvalidDir := t.TempDir() tempDirGitHub := t.TempDir() tempDirGitLab := t.TempDir() tempDirBitbucket := t.TempDir() - httpTimeout := 0 - invalidGitUrl := GitUrl{ Protocol: "", Host: "", @@ -403,8 +440,8 @@ func TestCloneGitRepo(t *testing.T) { token: "fake-github-token", } - privateRepoBadTokenErr := "failed to validate git url with token*" - publicRepoInvalidUrlErr := "failed to validate git url without a token" + privateRepoBadTokenErr := "failed to clone repo with token*" + publicRepoInvalidUrlErr := "failed to clone repo without a token" missingDestDirErr := "failed to clone repo, destination directory*" tests := []struct { @@ -450,7 +487,7 @@ func TestCloneGitRepo(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := CloneGitRepo(tt.gitUrl, tt.destDir, &httpTimeout) + err := CloneGitRepo(tt.gitUrl, tt.destDir) if (err != nil) != (tt.wantErr != "") { t.Errorf("Unxpected error: %t, want: %v", err, tt.wantErr) } else if err != nil { diff --git a/pkg/util/util.go b/pkg/util/util.go index f6c6fc90..d9f8b42b 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1091,6 +1091,21 @@ func DownloadInMemory(params HTTPRequestParams) ([]byte, error) { return nil, err } + if IsGitProviderRepo(url) { + g, err := ParseGitUrl(url) + if err != nil { + return nil, errors.Errorf("failed to parse git repo. error: %v", err) + } + if !g.IsPublic(params.Timeout) { + err = g.SetToken(params.Token, params.Timeout) + if err != nil { + return nil, err + } + bearer := "Bearer " + params.Token + req.Header.Add("Authorization", bearer) + } + } + //add the telemetry client name in the header req.Header.Add("Client", params.TelemetryClientName) resp, err := httpClient.Do(req) diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 31f83656..1de8bb1d 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -936,6 +936,11 @@ func TestDownloadFile(t *testing.T) { } } +//todo: +func TestDownloadInMemory(t *testing.T) { + +} + func TestValidateK8sResourceName(t *testing.T) { tests := []struct { name string From 6ba294593e77fb1d818dd4c17b67bb1dfad28bde Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Tue, 21 Mar 2023 09:02:05 -0400 Subject: [PATCH 05/15] cleanup Signed-off-by: Michael Hoang --- README.md | 4 +- pkg/devfile/parser/parse.go | 6 +- pkg/devfile/parser/parse_test.go | 2 +- pkg/util/git.go | 44 +++++++++-- pkg/util/git_test.go | 106 +++++++++++++++++-------- pkg/util/mock.go | 57 ++++++++++++++ pkg/util/util.go | 30 +++++-- pkg/util/util_test.go | 130 ++++++++++++++++++++++++++++++- 8 files changed, 325 insertions(+), 54 deletions(-) create mode 100644 pkg/util/mock.go diff --git a/README.md b/README.md index d62a1a43..cf67a1a1 100644 --- a/README.md +++ b/README.md @@ -198,10 +198,10 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g url = "https://gitlab.com//" // Parse the repo url - gitUrl, err := util.ParseGitUrl(url) + gitUrl, err := util.NewGitUrl(url) // Clone the repo to a destination dir - err = util.CloneGitRepo(gitUrl, destDir) + err = util.CloneGitRepo(*gitUrl, destDir) ``` ## Projects using devfile/library diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index f6fde6b7..59c05c9d 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -431,7 +431,7 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D d.Ctx = devfileCtx.NewURLDevfileCtx(newUri) if util.IsGitProviderRepo(newUri) { - gitUrl, err := util.ParseGitUrl(newUri) + gitUrl, err := util.NewGitUrl(newUri) if err != nil { return DevfileObj{}, err } @@ -450,7 +450,7 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D return populateAndParseDevfile(d, newResolveCtx, tool, true) } -func getResourcesFromGit(g util.GitUrl, destDir string, httpTimeout *int, repoToken string) error { +func getResourcesFromGit(g *util.GitUrl, destDir string, httpTimeout *int, repoToken string) error { stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) if err != nil { return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) @@ -464,7 +464,7 @@ func getResourcesFromGit(g util.GitUrl, destDir string, httpTimeout *int, repoTo } } - err = util.CloneGitRepo(g, stackDir) + err = util.CloneGitRepo(*g, stackDir) if err != nil { return err } diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index cf2ef303..7e78988e 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -4207,7 +4207,7 @@ func Test_getResourcesFromGit(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := getResourcesFromGit(tt.gitUrl, tt.destDir, &httpTimeout, "") + err := getResourcesFromGit(&tt.gitUrl, tt.destDir, &httpTimeout, "") if (err != nil) != tt.wantErr { t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err) } diff --git a/pkg/util/git.go b/pkg/util/git.go index ce548d0b..a27f97a0 100644 --- a/pkg/util/git.go +++ b/pkg/util/git.go @@ -31,6 +31,13 @@ const ( BitbucketHost string = "bitbucket.org" ) +type IGitUrl interface { + ParseGitUrl(fullUrl string) error + GitRawFileAPI() string + SetToken(token string, httpTimeout *int) error + IsPublic(httpTimeout *int) bool +} + type GitUrl struct { Protocol string // URL scheme Host string // URL domain name @@ -42,23 +49,30 @@ type GitUrl struct { IsFile bool // defines if the URL points to a file in the repo } +// NewGitUrl creates a GitUrl from a string url +func NewGitUrl(url string) (*GitUrl, error) { + g := &GitUrl{} + if err := g.ParseGitUrl(url); err != nil { + return g, err + } + return g, nil +} + // ParseGitUrl extracts information from a support git url // Only supports git repositories hosted on GitHub, GitLab, and Bitbucket -func ParseGitUrl(fullUrl string) (GitUrl, error) { - var g GitUrl - +func (g *GitUrl) ParseGitUrl(fullUrl string) error { err := ValidateURL(fullUrl) if err != nil { - return g, err + return err } parsedUrl, err := url.Parse(fullUrl) if err != nil { - return g, err + return err } if len(parsedUrl.Path) == 0 { - return g, fmt.Errorf("url path should not be empty") + return fmt.Errorf("url path should not be empty") } if parsedUrl.Host == RawGitHubHost || parsedUrl.Host == GitHubHost { @@ -71,7 +85,7 @@ func ParseGitUrl(fullUrl string) (GitUrl, error) { err = fmt.Errorf("url host should be a valid GitHub, GitLab, or Bitbucket host; received: %s", parsedUrl.Host) } - return g, err + return err } func (g *GitUrl) parseGitHubUrl(url *url.URL) error { @@ -242,6 +256,22 @@ func (g *GitUrl) validateToken(params HTTPRequestParams) error { return nil } +// GitRawFileAPI returns the endpoint for the git providers raw file +func (g *GitUrl) GitRawFileAPI() string { + var apiRawFile string + + switch g.Host { + case GitHubHost, RawGitHubHost: + apiRawFile = fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", g.Owner, g.Repo, g.Branch, g.Path) + case GitLabHost: + apiRawFile = fmt.Sprintf("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/files/%s/raw", g.Owner, g.Repo, g.Path) + case BitbucketHost: + apiRawFile = fmt.Sprintf("https://api.bitbucket.org/2.0/repositories/%s/%s/src/%s/%s", g.Owner, g.Repo, g.Branch, g.Path) + } + + return apiRawFile +} + // IsGitProviderRepo checks if the url matches a repo from a supported git provider func IsGitProviderRepo(url string) bool { if strings.Contains(url, RawGitHubHost) || strings.Contains(url, GitHubHost) || diff --git a/pkg/util/git_test.go b/pkg/util/git_test.go index 86435e00..04b291fc 100644 --- a/pkg/util/git_test.go +++ b/pkg/util/git_test.go @@ -30,11 +30,11 @@ var ( bitbucketToken = "fake-bitbucket-token" ) -func Test_ParseGitUrl(t *testing.T) { +func Test_NewGitUrl(t *testing.T) { tests := []struct { name string url string - wantUrl GitUrl + wantUrl *GitUrl wantErr string }{ { @@ -51,7 +51,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse public GitHub repo with root path", url: "https://github.com/devfile/library", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -70,7 +70,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse public GitHub repo with file path", url: "https://github.com/devfile/library/blob/main/devfile.yaml", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -84,7 +84,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse public GitHub repo with raw file path", url: "https://raw.githubusercontent.com/devfile/library/main/devfile.yaml", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "raw.githubusercontent.com", Owner: "devfile", @@ -108,7 +108,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse private GitHub repo with token", url: "https://github.com/fake-owner/fake-private-repo", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "github.com", Owner: "fake-owner", @@ -122,7 +122,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse private raw GitHub file path with token", url: "https://raw.githubusercontent.com/fake-owner/fake-private-repo/main/README.md", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "raw.githubusercontent.com", Owner: "fake-owner", @@ -137,7 +137,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse public GitLab repo with root path", url: "https://gitlab.com/gitlab-org/gitlab-foss", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "gitlab.com", Owner: "gitlab-org", @@ -156,7 +156,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse public GitLab repo with file path", url: "https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/README.md", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "gitlab.com", Owner: "gitlab-org", @@ -180,7 +180,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse private GitLab repo with token", url: "https://gitlab.com/fake-owner/fake-private-repo", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "gitlab.com", Owner: "fake-owner", @@ -194,7 +194,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse private raw GitLab file path with token", url: "https://gitlab.com/fake-owner/fake-private-repo/-/raw/main/README.md", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "gitlab.com", Owner: "fake-owner", @@ -209,7 +209,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse public Bitbucket repo with root path", url: "https://bitbucket.org/fake-owner/fake-public-repo", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -228,7 +228,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse public Bitbucket repo with file path", url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/README.md", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -242,7 +242,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse public Bitbucket file path with nested path", url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/directory/test.txt", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -256,7 +256,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse public Bitbucket repo with raw file path", url: "https://bitbucket.org/fake-owner/fake-public-repo/raw/main/README.md", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -285,7 +285,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse private Bitbucket repo with token", url: "https://bitbucket.org/fake-owner/fake-private-repo", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -299,7 +299,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse private raw Bitbucket file path with token", url: "https://bitbucket.org/fake-owner/fake-private-repo/raw/main/README.md", - wantUrl: GitUrl{ + wantUrl: &GitUrl{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -314,7 +314,7 @@ func Test_ParseGitUrl(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := ParseGitUrl(tt.url) + got, err := NewGitUrl(tt.url) if (err != nil) != (tt.wantErr != "") { t.Errorf("Unxpected error: %t, want: %v", err, tt.wantUrl) } else if err == nil && !reflect.DeepEqual(got, tt.wantUrl) { @@ -326,23 +326,63 @@ func Test_ParseGitUrl(t *testing.T) { } } -// todo: try mocking -func Test_SetToken(t *testing.T) { - g := GitUrl{ - Protocol: "https", - Host: "github.com", - Owner: "devfile", - Repo: "library", - Branch: "main", - token: "", +func Test_GetGitRawFileAPI(t *testing.T) { + tests := []struct { + name string + g GitUrl + want string + }{ + { + name: "Github url", + g: GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Branch: "main", + Path: "tests/README.md", + }, + want: "https://raw.githubusercontent.com/devfile/library/main/tests/README.md", + }, + { + name: "GitLab url", + g: GitUrl{ + Protocol: "https", + Host: "gitlab.com", + Owner: "gitlab-org", + Repo: "gitlab", + Branch: "master", + Path: "README.md", + }, + want: "https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab/repository/files/README.md/raw", + }, + { + name: "Bitbucket url", + g: GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "owner", + Repo: "repo-name", + Branch: "main", + Path: "path/to/file.md", + }, + want: "https://api.bitbucket.org/2.0/repositories/owner/repo-name/src/main/path/to/file.md", + }, + { + name: "Empty GitUrl", + g: GitUrl{}, + want: "", + }, } - httpTimeout := 0 - token := "fake-git-token" - - err := g.SetToken(token, &httpTimeout) - assert.NoError(t, err) - assert.Equal(t, token, g.token) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.g.GitRawFileAPI() + if !reflect.DeepEqual(result, tt.want) { + t.Errorf("Got: %v, want: %v", result, tt.want) + } + }) + } } func Test_IsPublic(t *testing.T) { diff --git a/pkg/util/mock.go b/pkg/util/mock.go new file mode 100644 index 00000000..061986cb --- /dev/null +++ b/pkg/util/mock.go @@ -0,0 +1,57 @@ +// +// Copyright 2023 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import "net/http" + +var ( + GetDoFunc func(req *http.Request) (*http.Response, error) + GetParseGitUrlFunc func(url string) error + GetGitRawFileAPIFunc func() string + GetSetTokenFunc func(token string, httpTimeout *int) error + GetIsPublicFunc func(httpTimeout *int) bool +) + +type MockClient struct { + DoFunc func(req *http.Request) (*http.Response, error) +} + +func (m *MockClient) Do(req *http.Request) (*http.Response, error) { + return GetDoFunc(req) +} + +type MockGitUrl struct { + ParseGitUrlFunc func(fullUrl string) error + GetGitRawFileAPIFunc func(url string) string + SetTokenFunc func(token string, httpTimeout *int) error + IsPublicFunc func(httpTimeout *int) bool +} + +func (m *MockGitUrl) ParseGitUrl(fullUrl string) error { + return GetParseGitUrlFunc(fullUrl) +} + +func (m *MockGitUrl) GitRawFileAPI() string { + return GetGitRawFileAPIFunc() +} + +func (m *MockGitUrl) SetToken(token string, httpTimeout *int) error { + return GetSetTokenFunc(token, httpTimeout) +} + +func (m *MockGitUrl) IsPublic(httpTimeout *int) bool { + return GetIsPublicFunc(httpTimeout) +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 3e1f9b84..432b92f0 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -54,6 +54,10 @@ import ( "k8s.io/klog" ) +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) +} + const ( HTTPRequestResponseTimeout = 30 * time.Second // HTTPRequestTimeout configures timeout of all HTTP requests ModeReadWriteFile = 0600 // default Permission for a file @@ -1080,29 +1084,43 @@ func DownloadFileInMemory(url string) ([]byte, error) { // DownloadInMemory uses HTTPRequestParams to download the file and return bytes func DownloadInMemory(params HTTPRequestParams) ([]byte, error) { - var httpClient = &http.Client{Transport: &http.Transport{ ResponseHeaderTimeout: HTTPRequestResponseTimeout, }, Timeout: HTTPRequestResponseTimeout} - url := params.URL + var err error + var gitUrl = &GitUrl{} + + if IsGitProviderRepo(params.URL) { + gitUrl, err = NewGitUrl(params.URL) + if err != nil { + return nil, errors.Errorf("failed to parse git repo. error: %v", err) + } + } + + return downloadInMemoryWithClient(params, httpClient, gitUrl) +} + +func downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient, g IGitUrl) ([]byte, error) { + var url string + url = params.URL req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } if IsGitProviderRepo(url) { - g, err := ParseGitUrl(url) + url = g.GitRawFileAPI() + req, err = http.NewRequest("GET", url, nil) if err != nil { - return nil, errors.Errorf("failed to parse git repo. error: %v", err) + return nil, err } if !g.IsPublic(params.Timeout) { err = g.SetToken(params.Token, params.Timeout) if err != nil { return nil, err } - bearer := "Bearer " + params.Token - req.Header.Add("Authorization", bearer) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", params.Token)) } } diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 1de8bb1d..420bff81 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -16,8 +16,11 @@ package util import ( + "bytes" "fmt" "github.com/devfile/library/v2/pkg/testingutil/filesystem" + "github.com/kylelemons/godebug/pretty" + "github.com/stretchr/testify/assert" "io/ioutil" corev1 "k8s.io/api/core/v1" "net" @@ -936,9 +939,132 @@ func TestDownloadFile(t *testing.T) { } } -//todo: +func TestDownloadInMemory_GitRepo(t *testing.T) { + respBody := []byte("test response body") + resp := &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader(respBody)), + } + + var Client = &MockClient{} + GetDoFunc = func(req *http.Request) (*http.Response, error) { + if req.Header.Get("Authorization") == "" { + return nil, fmt.Errorf("missing authorization header") + } + return resp, nil + } + + var GitUrlMock = &MockGitUrl{} + GetGitRawFileAPIFunc = func() string { + return "" + } + GetSetTokenFunc = func(token string, httpTimeout *int) error { + return nil + } + + tests := []struct { + name string + params HTTPRequestParams + GetIsPublicFunc func(httpTimeout *int) bool + want []byte + wantErr string + }{ + { + name: "Case 1: Private Github repo with token", + params: HTTPRequestParams{ + URL: "https://github.com/myorg/myrepo/file.txt", + Token: "fake-token", + }, + GetIsPublicFunc: func(httpTimeout *int) bool { return false }, + want: []byte("test response body"), + wantErr: "", + }, + { + name: "Case 2: Public Github repo without token", + params: HTTPRequestParams{ + URL: "https://github.com/myorg/myrepo/file.txt", + }, + GetIsPublicFunc: func(httpTimeout *int) bool { return true }, + want: []byte("test response body"), + wantErr: "missing authorization header", + }, + { + name: "Case 3: Non git provider repo", + params: HTTPRequestParams{ + URL: "https://repo.com/myorg/myrepo/file.txt", + Token: "", + }, + want: []byte("test response body"), + wantErr: "missing authorization header", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + GetIsPublicFunc = tt.GetIsPublicFunc + result, err := downloadInMemoryWithClient(tt.params, Client, GitUrlMock) + if (err != nil) != (tt.wantErr != "") { + t.Errorf("Unxpected error: %t, want: %v", err, tt.want) + } else if err == nil && !reflect.DeepEqual(result, tt.want) { + t.Errorf("Expected: %v, received: %v, difference at %v", tt.want, result, pretty.Compare(tt.want, result)) + } else if err != nil { + assert.Regexp(t, tt.wantErr, err.Error(), "Error message should match") + } + }) + } +} + func TestDownloadInMemory(t *testing.T) { - + // Start a local HTTP server + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + // Send response to be tested + _, err := rw.Write([]byte("OK")) + if err != nil { + t.Error(err) + } + })) + + // Close the server when test finishes + defer server.Close() + + tests := []struct { + name string + url string + token string + want []byte + wantErr string + }{ + { + name: "Case 1: Input url is valid", + url: server.URL, + want: []byte{79, 75}, + }, + { + name: "Case 2: Input url is invalid", + url: "invalid", + wantErr: "unsupported protocol scheme", + }, + { + name: "Case 3: Git provider with invalid url", + url: "github.com/mike-hoang/invalid-repo", + token: "", + want: []byte(nil), + wantErr: "failed to parse git repo. error:*", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := DownloadInMemory(HTTPRequestParams{URL: tt.url, Token: tt.token}) + if (err != nil) != (tt.wantErr != "") { + t.Errorf("Failed to download file with error: %s", err) + } else if err == nil && !reflect.DeepEqual(data, tt.want) { + t.Errorf("Expected: %v, received: %v, difference at %v", tt.want, string(data[:]), pretty.Compare(tt.want, data)) + } else if err != nil { + assert.Regexp(t, tt.wantErr, err.Error(), "Error message should match") + } + }) + } } func TestValidateK8sResourceName(t *testing.T) { From 9201a29d604bd25fb1da5551bc62d127fd0eebe6 Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Wed, 29 Mar 2023 10:12:13 -0400 Subject: [PATCH 06/15] adding token processing for private main devfile Signed-off-by: Michael Hoang --- README.md | 23 ++------ pkg/devfile/parser/context/content.go | 3 + pkg/devfile/parser/context/context.go | 16 +++++ pkg/devfile/parser/parse.go | 17 ++++-- pkg/devfile/parser/parse_test.go | 84 +++++++++++++++++++++++++++ pkg/util/git_test.go | 6 -- 6 files changed, 121 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index cf67a1a1..df6aab78 100644 --- a/README.md +++ b/README.md @@ -16,21 +16,22 @@ The Devfile Parser library is a Golang module that: ## Private Repository Support Tokens are required to be set in the following cases: -1. tooling client calling the library API -2. parsing a devfile from a private repository -3. parsing a devfile containing a parent devfile from a private repository [1] +1. parsing a devfile from a private repository +2. parsing a devfile containing a parent devfile from a private repository [1] Set the token for the repository: ```go parser.ParserArgs{ ... + URL: Token: ... } ``` +Note: The url must also be set with a supported git provider repo url. For more information about personal access tokens: -1. [GitHub docs](https://docs.github.com/en/enterprise-server@3.4/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) +1. [GitHub docs](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) 2. [GitLab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token) 3. [Bitbucket docs](https://support.atlassian.com/bitbucket-cloud/docs/repository-access-tokens/) @@ -190,20 +191,6 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g resources, err := ParseKubernetesYaml(values) ``` -8. To get resources from Git repositories - ```go - // Supported url formats - url = "https://github.com//" - url = "https://bitbucket.org//" - url = "https://gitlab.com//" - - // Parse the repo url - gitUrl, err := util.NewGitUrl(url) - - // Clone the repo to a destination dir - err = util.CloneGitRepo(*gitUrl, destDir) - ``` - ## Projects using devfile/library The following projects are consuming this library as a Golang dependency diff --git a/pkg/devfile/parser/context/content.go b/pkg/devfile/parser/context/content.go index 3bab1fa0..1347b75a 100644 --- a/pkg/devfile/parser/context/content.go +++ b/pkg/devfile/parser/context/content.go @@ -69,6 +69,9 @@ func (d *DevfileCtx) SetDevfileContent() error { if d.url != "" { // set the client identifier for telemetry params := util.HTTPRequestParams{URL: d.url, TelemetryClientName: util.TelemetryClientName} + if d.token != "" { + params.Token = d.token + } data, err = util.DownloadInMemory(params) if err != nil { return errors.Wrap(err, "error getting devfile info from url") diff --git a/pkg/devfile/parser/context/context.go b/pkg/devfile/parser/context/context.go index bd2e6d6d..54f397c5 100644 --- a/pkg/devfile/parser/context/context.go +++ b/pkg/devfile/parser/context/context.go @@ -48,6 +48,9 @@ type DevfileCtx struct { //url path of the devfile url string + // token is a personal access token used with a private git repo URL + token string + // filesystem for devfile fs filesystem.Filesystem @@ -70,6 +73,14 @@ func NewURLDevfileCtx(url string) DevfileCtx { } } +// NewPrivateURLDevfileCtx returns a new DevfileCtx type object +func NewPrivateURLDevfileCtx(url string, token string) DevfileCtx { + return DevfileCtx{ + url: url, + token: token, + } +} + // NewByteContentDevfileCtx set devfile content from byte data and returns a new DevfileCtx type object and error func NewByteContentDevfileCtx(data []byte) (d DevfileCtx, err error) { err = d.SetDevfileContentFromBytes(data) @@ -150,6 +161,11 @@ func (d *DevfileCtx) GetURL() string { return d.url } +// GetToken func returns current devfile token +func (d *DevfileCtx) GetToken() string { + return d.token +} + // SetAbsPath sets absolute file path for devfile func (d *DevfileCtx) SetAbsPath() (err error) { // Set devfile absolute path diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index 59c05c9d..544c6b7c 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -97,7 +97,7 @@ type ParserArgs struct { // RegistryURLs is a list of registry hosts which parser should pull parent devfile from. // If registryUrl is defined in devfile, this list will be ignored. RegistryURLs []string - // Token is a GitHub, GitLab, or Bitbucket personal access token used with a private git repo uri + // Token is a GitHub, GitLab, or Bitbucket personal access token used with a private git repo URL Token string // DefaultNamespace is the default namespace to use // If namespace is defined under devfile's parent kubernetes object, this namespace will be ignored. @@ -123,7 +123,11 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) { } else if args.Path != "" { d.Ctx = devfileCtx.NewDevfileCtx(args.Path) } else if args.URL != "" { - d.Ctx = devfileCtx.NewURLDevfileCtx(args.URL) + if args.Token != "" { + d.Ctx = devfileCtx.NewPrivateURLDevfileCtx(args.URL, args.Token) + } else { + d.Ctx = devfileCtx.NewURLDevfileCtx(args.URL) + } } else { return d, errors.Wrap(err, "the devfile source is not provided") } @@ -178,7 +182,7 @@ type resolverTools struct { // RegistryURLs is a list of registry hosts which parser should pull parent devfile from. // If registryUrl is defined in devfile, this list will be ignored. registryURLs []string - // Token is a GitHub, GitLab, or Bitbucket personal access token used with a private git repo uri + // Token is a GitHub, GitLab, or Bitbucket personal access token used with a private git repo URL token string // Context is the context used for making Kubernetes or HTTP requests context context.Context @@ -429,7 +433,12 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D return DevfileObj{}, fmt.Errorf("failed to resolve parent uri, devfile context is missing absolute url and path to devfile. %s", resolveImportReference(importReference)) } - d.Ctx = devfileCtx.NewURLDevfileCtx(newUri) + if tool.token != "" { + d.Ctx = devfileCtx.NewPrivateURLDevfileCtx(newUri, tool.token) + } else { + d.Ctx = devfileCtx.NewURLDevfileCtx(newUri) + } + if util.IsGitProviderRepo(newUri) { gitUrl, err := util.NewGitUrl(newUri) if err != nil { diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index 7e78988e..a586e755 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -3825,6 +3825,90 @@ func Test_parseFromURI(t *testing.T) { } } +func Test_parseFromURI_PrivateToken(t *testing.T) { + curDevfileContext := devfileCtx.NewDevfileCtx(OutputDevfileYamlPath) + + nestedParent := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + _, err := rw.Write([]byte("schemaVersion: 2.2.0")) + if err != nil { + t.Error(err) + } + })) + + parentDevfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: nested-devfile\nparent:\n uri: \"%s\"", nestedParent.URL) + parent := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + _, err := rw.Write([]byte(parentDevfileContent)) + if err != nil { + t.Error(err) + } + })) + + devfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: parent-devfile\nparent:\n uri: \"%s\"", parent.URL) + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + _, err := rw.Write([]byte(devfileContent)) + if err != nil { + t.Error(err) + } + })) + + // Close the server when test finishes + defer server.Close() + defer parent.Close() + defer nestedParent.Close() + + tests := []struct { + name string + curDevfileCtx devfileCtx.DevfileCtx + importReference v1.ImportReference + token string + wantToken string + }{ + { + name: "Nested parent: token should be set", + curDevfileCtx: curDevfileContext, + token: "fake-token", + wantToken: "fake-token", + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: server.URL, + }, + }, + }, + { + name: "Parent: token should be set", + curDevfileCtx: curDevfileContext, + token: "fake-token", + wantToken: "fake-token", + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: parent.URL, + }, + }, + }, + { + name: "Main devfile: token should be set", + curDevfileCtx: curDevfileContext, + token: "fake-token", + wantToken: "fake-token", + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: nestedParent.URL, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseFromURI(tt.importReference, tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{token: tt.token}) + if !reflect.DeepEqual(tt.wantToken, got.Ctx.GetToken()) { + t.Errorf("Expected %s, got %s", tt.wantToken, got.Ctx.GetToken()) + } else if err != nil { + t.Errorf("Unxpected error: %t", err) + } + }) + } +} + func Test_parseFromRegistry(t *testing.T) { const ( registry = "127.0.0.1:8080" diff --git a/pkg/util/git_test.go b/pkg/util/git_test.go index 04b291fc..a2e14023 100644 --- a/pkg/util/git_test.go +++ b/pkg/util/git_test.go @@ -24,12 +24,6 @@ import ( "testing" ) -var ( - githubToken = "fake-github-token" - gitlabToken = "fake-gitlab-token" - bitbucketToken = "fake-bitbucket-token" -) - func Test_NewGitUrl(t *testing.T) { tests := []struct { name string From 06230f27c06c9addf77e3b48ea1a59a6d85fa974 Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Thu, 30 Mar 2023 11:33:24 -0400 Subject: [PATCH 07/15] cleanup test Signed-off-by: Michael Hoang --- pkg/util/git_test.go | 111 ++++--------------------------------------- 1 file changed, 9 insertions(+), 102 deletions(-) diff --git a/pkg/util/git_test.go b/pkg/util/git_test.go index a2e14023..97ab982e 100644 --- a/pkg/util/git_test.go +++ b/pkg/util/git_test.go @@ -43,7 +43,7 @@ func Test_NewGitUrl(t *testing.T) { }, // GitHub { - name: "should parse public GitHub repo with root path", + name: "should parse GitHub repo with root path", url: "https://github.com/devfile/library", wantUrl: &GitUrl{ Protocol: "https", @@ -52,7 +52,6 @@ func Test_NewGitUrl(t *testing.T) { Repo: "library", Branch: "", Path: "", - token: "", IsFile: false, }, }, @@ -62,7 +61,7 @@ func Test_NewGitUrl(t *testing.T) { wantErr: "url path should contain /*", }, { - name: "should parse public GitHub repo with file path", + name: "should parse GitHub repo with file path", url: "https://github.com/devfile/library/blob/main/devfile.yaml", wantUrl: &GitUrl{ Protocol: "https", @@ -71,12 +70,11 @@ func Test_NewGitUrl(t *testing.T) { Repo: "library", Branch: "main", Path: "devfile.yaml", - token: "", IsFile: true, }, }, { - name: "should parse public GitHub repo with raw file path", + name: "should parse GitHub repo with raw file path", url: "https://raw.githubusercontent.com/devfile/library/main/devfile.yaml", wantUrl: &GitUrl{ Protocol: "https", @@ -85,7 +83,6 @@ func Test_NewGitUrl(t *testing.T) { Repo: "library", Branch: "main", Path: "devfile.yaml", - token: "", IsFile: true, }, }, @@ -99,37 +96,9 @@ func Test_NewGitUrl(t *testing.T) { url: "https://raw.githubusercontent.com/devfile/library/devfile.yaml", wantErr: "raw url path should contain ///*", }, - { - name: "should parse private GitHub repo with token", - url: "https://github.com/fake-owner/fake-private-repo", - wantUrl: &GitUrl{ - Protocol: "https", - Host: "github.com", - Owner: "fake-owner", - Repo: "fake-private-repo", - Branch: "", - Path: "", - token: "", - IsFile: false, - }, - }, - { - name: "should parse private raw GitHub file path with token", - url: "https://raw.githubusercontent.com/fake-owner/fake-private-repo/main/README.md", - wantUrl: &GitUrl{ - Protocol: "https", - Host: "raw.githubusercontent.com", - Owner: "fake-owner", - Repo: "fake-private-repo", - Branch: "main", - Path: "README.md", - token: "", - IsFile: true, - }, - }, // Gitlab { - name: "should parse public GitLab repo with root path", + name: "should parse GitLab repo with root path", url: "https://gitlab.com/gitlab-org/gitlab-foss", wantUrl: &GitUrl{ Protocol: "https", @@ -138,7 +107,6 @@ func Test_NewGitUrl(t *testing.T) { Repo: "gitlab-foss", Branch: "", Path: "", - token: "", IsFile: false, }, }, @@ -148,7 +116,7 @@ func Test_NewGitUrl(t *testing.T) { wantErr: "url path should contain /*", }, { - name: "should parse public GitLab repo with file path", + name: "should parse GitLab repo with file path", url: "https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/README.md", wantUrl: &GitUrl{ Protocol: "https", @@ -157,7 +125,6 @@ func Test_NewGitUrl(t *testing.T) { Repo: "gitlab-foss", Branch: "master", Path: "README.md", - token: "", IsFile: true, }, }, @@ -171,37 +138,9 @@ func Test_NewGitUrl(t *testing.T) { url: "https://gitlab.com/gitlab-org/gitlab-foss/-/master/directory/README.md", wantErr: "url path should contain 'blob' or 'tree' or 'raw'*", }, - { - name: "should parse private GitLab repo with token", - url: "https://gitlab.com/fake-owner/fake-private-repo", - wantUrl: &GitUrl{ - Protocol: "https", - Host: "gitlab.com", - Owner: "fake-owner", - Repo: "fake-private-repo", - Branch: "", - Path: "", - token: "", - IsFile: false, - }, - }, - { - name: "should parse private raw GitLab file path with token", - url: "https://gitlab.com/fake-owner/fake-private-repo/-/raw/main/README.md", - wantUrl: &GitUrl{ - Protocol: "https", - Host: "gitlab.com", - Owner: "fake-owner", - Repo: "fake-private-repo", - Branch: "main", - Path: "README.md", - token: "", - IsFile: true, - }, - }, // Bitbucket { - name: "should parse public Bitbucket repo with root path", + name: "should parse Bitbucket repo with root path", url: "https://bitbucket.org/fake-owner/fake-public-repo", wantUrl: &GitUrl{ Protocol: "https", @@ -210,7 +149,6 @@ func Test_NewGitUrl(t *testing.T) { Repo: "fake-public-repo", Branch: "", Path: "", - token: "", IsFile: false, }, }, @@ -220,7 +158,7 @@ func Test_NewGitUrl(t *testing.T) { wantErr: "url path should contain /*", }, { - name: "should parse public Bitbucket repo with file path", + name: "should parse Bitbucket repo with file path", url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/README.md", wantUrl: &GitUrl{ Protocol: "https", @@ -229,12 +167,11 @@ func Test_NewGitUrl(t *testing.T) { Repo: "fake-public-repo", Branch: "main", Path: "README.md", - token: "", IsFile: true, }, }, { - name: "should parse public Bitbucket file path with nested path", + name: "should parse Bitbucket file path with nested path", url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/directory/test.txt", wantUrl: &GitUrl{ Protocol: "https", @@ -243,12 +180,11 @@ func Test_NewGitUrl(t *testing.T) { Repo: "fake-public-repo", Branch: "main", Path: "directory/test.txt", - token: "", IsFile: true, }, }, { - name: "should parse public Bitbucket repo with raw file path", + name: "should parse Bitbucket repo with raw file path", url: "https://bitbucket.org/fake-owner/fake-public-repo/raw/main/README.md", wantUrl: &GitUrl{ Protocol: "https", @@ -257,7 +193,6 @@ func Test_NewGitUrl(t *testing.T) { Repo: "fake-public-repo", Branch: "main", Path: "README.md", - token: "", IsFile: true, }, }, @@ -276,34 +211,6 @@ func Test_NewGitUrl(t *testing.T) { url: "https://bitbucket.org/fake-owner/fake-public-repo/main/test/README.md", wantErr: "url path should contain 'raw' or 'src'*", }, - { - name: "should parse private Bitbucket repo with token", - url: "https://bitbucket.org/fake-owner/fake-private-repo", - wantUrl: &GitUrl{ - Protocol: "https", - Host: "bitbucket.org", - Owner: "fake-owner", - Repo: "fake-private-repo", - Branch: "", - Path: "", - token: "", - IsFile: false, - }, - }, - { - name: "should parse private raw Bitbucket file path with token", - url: "https://bitbucket.org/fake-owner/fake-private-repo/raw/main/README.md", - wantUrl: &GitUrl{ - Protocol: "https", - Host: "bitbucket.org", - Owner: "fake-owner", - Repo: "fake-private-repo", - Branch: "main", - Path: "README.md", - token: "", - IsFile: true, - }, - }, } for _, tt := range tests { From 3bf66d527a6f4262a1cf81fc66e6c405dd55b1dc Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Mon, 3 Apr 2023 15:28:14 -0400 Subject: [PATCH 08/15] getting token from devfile ctx and update errors on parsing Signed-off-by: Michael Hoang --- README.md | 4 +- pkg/devfile/parse_test.go | 86 ++++++++++++++++++++++ pkg/devfile/parser/context/context_test.go | 18 +++++ pkg/devfile/parser/parse.go | 10 +-- pkg/devfile/parser/parse_test.go | 84 --------------------- pkg/util/git.go | 43 ++++++++--- pkg/util/git_test.go | 25 +++++++ pkg/util/util_test.go | 9 ++- 8 files changed, 174 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index 6cf21215..f5352480 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The Devfile Parser library is a Golang module that: Tokens are required to be set in the following cases: 1. parsing a devfile from a private repository 2. parsing a devfile containing a parent devfile from a private repository [1] +3. parsing a devfile from a private repository containing a parent devfile from a public repository [2] Set the token for the repository: ```go @@ -35,7 +36,8 @@ For more information about personal access tokens: 2. [GitLab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token) 3. [Bitbucket docs](https://support.atlassian.com/bitbucket-cloud/docs/repository-access-tokens/) -[1] currently, this works under the assumption that the token can authenticate the devfile and the parent devfile; both devfiles are in the same repository. +[1] Currently, this works under the assumption that the token can authenticate the devfile and the parent devfile; both devfiles are in the same repository. +[2] In this scenario, the token will be used to authenticate the main devfile. ## Usage diff --git a/pkg/devfile/parse_test.go b/pkg/devfile/parse_test.go index 3fe471fb..bfc3f7d8 100644 --- a/pkg/devfile/parse_test.go +++ b/pkg/devfile/parse_test.go @@ -16,6 +16,7 @@ package devfile import ( + "fmt" "net" "net/http" "net/http/httptest" @@ -416,3 +417,88 @@ schemaVersion: 2.2.0 }) } } + +// TestParseDevfileAndValidate_PrivateTokens checks that tokens are passed along from the parser args +// to nested parent devfiles +func TestParseDevfileAndValidate_PrivateTokens(t *testing.T) { + nestedParent := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + _, err := rw.Write([]byte("schemaVersion: 2.2.0")) + if err != nil { + t.Error(err) + } + })) + + parentDevfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: nested-devfile\nparent:\n uri: \"%s\"", nestedParent.URL) + parent := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + _, err := rw.Write([]byte(parentDevfileContent)) + if err != nil { + t.Error(err) + } + })) + + devfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: parent-devfile\nparent:\n uri: \"%s\"", parent.URL) + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + _, err := rw.Write([]byte(devfileContent)) + if err != nil { + t.Error(err) + } + })) + + // Close the server when test finishes + defer server.Close() + defer parent.Close() + defer nestedParent.Close() + + token := "fake-token" + + type args struct { + args parser.ParserArgs + } + tests := []struct { + name string + args args + wantToken string + }{ + { + name: "Nested parent: token should be set", + args: args{ + args: parser.ParserArgs{ + URL: server.URL, + Token: "fake-token", + }, + }, + wantToken: token, + }, + { + name: "Parent: token should be set", + args: args{ + args: parser.ParserArgs{ + URL: parent.URL, + Token: "fake-token", + }, + }, + wantToken: token, + }, + { + name: "Main devfile: token should be set", + args: args{ + args: parser.ParserArgs{ + URL: nestedParent.URL, + Token: "fake-token", + }, + }, + wantToken: token, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotD, _, err := ParseDevfileAndValidate(tt.args.args) + fmt.Println("gotD: ", string(gotD.Ctx.GetDevfileContent())) + if !reflect.DeepEqual(tt.wantToken, gotD.Ctx.GetToken()) { + t.Errorf("Expected %s, got %s", tt.wantToken, gotD.Ctx.GetToken()) + } else if err != nil { + t.Errorf("Unxpected error: %t", err) + } + }) + } +} diff --git a/pkg/devfile/parser/context/context_test.go b/pkg/devfile/parser/context/context_test.go index 9fe14cd9..32097edd 100644 --- a/pkg/devfile/parser/context/context_test.go +++ b/pkg/devfile/parser/context/context_test.go @@ -83,6 +83,24 @@ func TestPopulateFromInvalidURL(t *testing.T) { }) } +func TestNewURLDevfileCtx(t *testing.T) { + var ( + token = "fake-token" + url = "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml" + ) + + { + d := NewPrivateURLDevfileCtx(url, token) + assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL()) + assert.Equal(t, "fake-token", d.GetToken()) + } + { + d := NewURLDevfileCtx(url) + assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL()) + assert.Equal(t, "", d.GetToken()) + } +} + func invalidJsonRawContent200() []byte { return []byte(InvalidDevfileContent) } diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index ace84073..e7bea9ce 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -141,7 +141,6 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) { context: args.Context, k8sClient: args.K8sClient, httpTimeout: args.HTTPTimeout, - token: args.Token, } flattenedDevfile := true @@ -189,8 +188,6 @@ type resolverTools struct { // RegistryURLs is a list of registry hosts which parser should pull parent devfile from. // If registryUrl is defined in devfile, this list will be ignored. registryURLs []string - // Token is a GitHub, GitLab, or Bitbucket personal access token used with a private git repo URL - token string // Context is the context used for making Kubernetes or HTTP requests context context.Context // K8sClient is the Kubernetes client instance used for interacting with a cluster @@ -440,8 +437,9 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D return DevfileObj{}, fmt.Errorf("failed to resolve parent uri, devfile context is missing absolute url and path to devfile. %s", resolveImportReference(importReference)) } - if tool.token != "" { - d.Ctx = devfileCtx.NewPrivateURLDevfileCtx(newUri, tool.token) + token := curDevfileCtx.GetToken() + if token != "" { + d.Ctx = devfileCtx.NewPrivateURLDevfileCtx(newUri, token) } else { d.Ctx = devfileCtx.NewURLDevfileCtx(newUri) } @@ -453,7 +451,7 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D } if gitUrl.IsFile { destDir := path.Dir(curDevfileCtx.GetAbsPath()) - err = getResourcesFromGit(gitUrl, destDir, tool.httpTimeout, tool.token) + err = getResourcesFromGit(gitUrl, destDir, tool.httpTimeout, token) if err != nil { return DevfileObj{}, err } diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index d1770790..c2266e00 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -4156,90 +4156,6 @@ func Test_parseFromURI(t *testing.T) { } } -func Test_parseFromURI_PrivateToken(t *testing.T) { - curDevfileContext := devfileCtx.NewDevfileCtx(OutputDevfileYamlPath) - - nestedParent := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - _, err := rw.Write([]byte("schemaVersion: 2.2.0")) - if err != nil { - t.Error(err) - } - })) - - parentDevfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: nested-devfile\nparent:\n uri: \"%s\"", nestedParent.URL) - parent := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - _, err := rw.Write([]byte(parentDevfileContent)) - if err != nil { - t.Error(err) - } - })) - - devfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: parent-devfile\nparent:\n uri: \"%s\"", parent.URL) - server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - _, err := rw.Write([]byte(devfileContent)) - if err != nil { - t.Error(err) - } - })) - - // Close the server when test finishes - defer server.Close() - defer parent.Close() - defer nestedParent.Close() - - tests := []struct { - name string - curDevfileCtx devfileCtx.DevfileCtx - importReference v1.ImportReference - token string - wantToken string - }{ - { - name: "Nested parent: token should be set", - curDevfileCtx: curDevfileContext, - token: "fake-token", - wantToken: "fake-token", - importReference: v1.ImportReference{ - ImportReferenceUnion: v1.ImportReferenceUnion{ - Uri: server.URL, - }, - }, - }, - { - name: "Parent: token should be set", - curDevfileCtx: curDevfileContext, - token: "fake-token", - wantToken: "fake-token", - importReference: v1.ImportReference{ - ImportReferenceUnion: v1.ImportReferenceUnion{ - Uri: parent.URL, - }, - }, - }, - { - name: "Main devfile: token should be set", - curDevfileCtx: curDevfileContext, - token: "fake-token", - wantToken: "fake-token", - importReference: v1.ImportReference{ - ImportReferenceUnion: v1.ImportReferenceUnion{ - Uri: nestedParent.URL, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := parseFromURI(tt.importReference, tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{token: tt.token}) - if !reflect.DeepEqual(tt.wantToken, got.Ctx.GetToken()) { - t.Errorf("Expected %s, got %s", tt.wantToken, got.Ctx.GetToken()) - } else if err != nil { - t.Errorf("Unxpected error: %t", err) - } - }) - } -} - func Test_parseFromRegistry(t *testing.T) { const ( registry = "127.0.0.1:8080" diff --git a/pkg/util/git.go b/pkg/util/git.go index a27f97a0..30a9edfa 100644 --- a/pkg/util/git.go +++ b/pkg/util/git.go @@ -105,8 +105,10 @@ func (g *GitUrl) parseGitHubUrl(url *url.URL) error { g.Branch = splitUrl[2] g.Path = splitUrl[3] } else { + // raw GitHub urls have to be a file err = fmt.Errorf("raw url path should contain ///, received: %s", url.Path[1:]) } + return err } if g.Host == GitHubHost { @@ -117,15 +119,26 @@ func (g *GitUrl) parseGitHubUrl(url *url.URL) error { g.Owner = splitUrl[0] g.Repo = splitUrl[1] + // url doesn't contain a path to a directory or file + if len(splitUrl) == 2 { + return nil + } + + switch splitUrl[2] { + case "tree": + g.IsFile = false + case "blob": + g.IsFile = true + default: + return fmt.Errorf("url path to directory or file should contain 'tree' or 'blob'") + } + + // url has a path to a file or directory if len(splitUrl) == 5 { - switch splitUrl[2] { - case "tree": - g.IsFile = false - case "blob": - g.IsFile = true - } g.Branch = splitUrl[3] g.Path = splitUrl[4] + } else { + err = fmt.Errorf("url path should contain ////, received: %s", url.Path[1:]) } } } @@ -146,17 +159,23 @@ func (g *GitUrl) parseGitLabUrl(url *url.URL) error { split := strings.Split(url.Path[1:], "/-/") splitOrg = strings.SplitN(split[0], "/", 2) - if len(split) == 2 { - splitFile = strings.SplitN(split[1], "/", 3) - } - if len(splitOrg) < 2 { - err = fmt.Errorf("url path should contain /, received: %s", url.Path[1:]) + return fmt.Errorf("url path should contain /, received: %s", url.Path[1:]) } else { g.Owner = splitOrg[0] g.Repo = splitOrg[1] } + // url doesn't contain a path to a directory or file + if len(split) == 1 { + return nil + } + + // url may contain a path to a directory or file + if len(split) == 2 { + splitFile = strings.SplitN(split[1], "/", 3) + } + if len(splitFile) == 3 { if splitFile[0] == "blob" || splitFile[0] == "tree" || splitFile[0] == "raw" { g.Branch = splitFile[1] @@ -168,6 +187,8 @@ func (g *GitUrl) parseGitLabUrl(url *url.URL) error { } else { err = fmt.Errorf("url path should contain 'blob' or 'tree' or 'raw', received: %s", url.Path[1:]) } + } else { + return fmt.Errorf("url path to directory or file should contain //, received: %s", url.Path[1:]) } return err diff --git a/pkg/util/git_test.go b/pkg/util/git_test.go index 97ab982e..d7773fdf 100644 --- a/pkg/util/git_test.go +++ b/pkg/util/git_test.go @@ -91,6 +91,26 @@ func Test_NewGitUrl(t *testing.T) { url: "https://github.com/devfile", wantErr: "url path should contain /*", }, + { + name: "should fail with missing GitHub blob", + url: "https://github.com/devfile/library/main/devfile.yaml", + wantErr: "url path to directory or file should contain*", + }, + { + name: "should fail with missing GitHub tree", + url: "https://github.com/devfile/library/main/tests/yamls", + wantErr: "url path to directory or file should contain*", + }, + { + name: "should fail with just GitHub tree", + url: "https://github.com/devfile/library/tree", + wantErr: "url path should contain ////*", + }, + { + name: "should fail with just GitHub blob", + url: "https://github.com/devfile/library/blob", + wantErr: "url path should contain ////*", + }, { name: "should fail with invalid GitHub raw file path", url: "https://raw.githubusercontent.com/devfile/library/devfile.yaml", @@ -138,6 +158,11 @@ func Test_NewGitUrl(t *testing.T) { url: "https://gitlab.com/gitlab-org/gitlab-foss/-/master/directory/README.md", wantErr: "url path should contain 'blob' or 'tree' or 'raw'*", }, + { + name: "should fail with missing GitLab file or directory path", + url: "https://gitlab.com/gitlab-org/gitlab-foss/-/tree/master", + wantErr: "url path to directory or file should contain //*", + }, // Bitbucket { name: "should parse Bitbucket repo with root path", diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 420bff81..c89466c6 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -980,12 +980,11 @@ func TestDownloadInMemory_GitRepo(t *testing.T) { wantErr: "", }, { - name: "Case 2: Public Github repo without token", + name: "Case 2: Private Github repo without token", params: HTTPRequestParams{ URL: "https://github.com/myorg/myrepo/file.txt", }, GetIsPublicFunc: func(httpTimeout *int) bool { return true }, - want: []byte("test response body"), wantErr: "missing authorization header", }, { @@ -994,7 +993,6 @@ func TestDownloadInMemory_GitRepo(t *testing.T) { URL: "https://repo.com/myorg/myrepo/file.txt", Token: "", }, - want: []byte("test response body"), wantErr: "missing authorization header", }, } @@ -1051,6 +1049,11 @@ func TestDownloadInMemory(t *testing.T) { want: []byte(nil), wantErr: "failed to parse git repo. error:*", }, + { + name: "Case 4: Public Github repo with missing blob", + url: "https://github.com/devfile/library/main/README.md", + wantErr: "failed to parse git repo. error: url path to directory or file should contain 'tree' or 'blob'*", + }, } for _, tt := range tests { From 8a33b339acb8d9e503a0727ae663bfa4699ed603 Mon Sep 17 00:00:00 2001 From: Michael Hoang <35011707+mike-hoang@users.noreply.github.com> Date: Fri, 14 Apr 2023 07:23:21 -0400 Subject: [PATCH 09/15] adding mocks for git Signed-off-by: Michael Hoang --- pkg/devfile/parse_test.go | 86 ------- pkg/devfile/parser/context/context.go | 20 +- pkg/devfile/parser/context/context_test.go | 3 + pkg/devfile/parser/parse.go | 51 +--- pkg/devfile/parser/parse_test.go | 230 +++++++++++++++--- pkg/{util => git}/git.go | 195 ++++++++++------ pkg/{util => git}/git_test.go | 82 ++++--- pkg/git/mock.go | 136 +++++++++++ pkg/git/util.go | 260 +++++++++++++++++++++ pkg/git/util_test.go | 122 ++++++++++ pkg/util/mock.go | 57 ----- pkg/util/util.go | 18 +- pkg/util/util_test.go | 74 ------ 13 files changed, 946 insertions(+), 388 deletions(-) rename pkg/{util => git}/git.go (72%) rename pkg/{util => git}/git_test.go (90%) create mode 100644 pkg/git/mock.go create mode 100644 pkg/git/util.go create mode 100644 pkg/git/util_test.go delete mode 100644 pkg/util/mock.go diff --git a/pkg/devfile/parse_test.go b/pkg/devfile/parse_test.go index bfc3f7d8..3fe471fb 100644 --- a/pkg/devfile/parse_test.go +++ b/pkg/devfile/parse_test.go @@ -16,7 +16,6 @@ package devfile import ( - "fmt" "net" "net/http" "net/http/httptest" @@ -417,88 +416,3 @@ schemaVersion: 2.2.0 }) } } - -// TestParseDevfileAndValidate_PrivateTokens checks that tokens are passed along from the parser args -// to nested parent devfiles -func TestParseDevfileAndValidate_PrivateTokens(t *testing.T) { - nestedParent := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - _, err := rw.Write([]byte("schemaVersion: 2.2.0")) - if err != nil { - t.Error(err) - } - })) - - parentDevfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: nested-devfile\nparent:\n uri: \"%s\"", nestedParent.URL) - parent := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - _, err := rw.Write([]byte(parentDevfileContent)) - if err != nil { - t.Error(err) - } - })) - - devfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: parent-devfile\nparent:\n uri: \"%s\"", parent.URL) - server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - _, err := rw.Write([]byte(devfileContent)) - if err != nil { - t.Error(err) - } - })) - - // Close the server when test finishes - defer server.Close() - defer parent.Close() - defer nestedParent.Close() - - token := "fake-token" - - type args struct { - args parser.ParserArgs - } - tests := []struct { - name string - args args - wantToken string - }{ - { - name: "Nested parent: token should be set", - args: args{ - args: parser.ParserArgs{ - URL: server.URL, - Token: "fake-token", - }, - }, - wantToken: token, - }, - { - name: "Parent: token should be set", - args: args{ - args: parser.ParserArgs{ - URL: parent.URL, - Token: "fake-token", - }, - }, - wantToken: token, - }, - { - name: "Main devfile: token should be set", - args: args{ - args: parser.ParserArgs{ - URL: nestedParent.URL, - Token: "fake-token", - }, - }, - wantToken: token, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotD, _, err := ParseDevfileAndValidate(tt.args.args) - fmt.Println("gotD: ", string(gotD.Ctx.GetDevfileContent())) - if !reflect.DeepEqual(tt.wantToken, gotD.Ctx.GetToken()) { - t.Errorf("Expected %s, got %s", tt.wantToken, gotD.Ctx.GetToken()) - } else if err != nil { - t.Errorf("Unxpected error: %t", err) - } - }) - } -} diff --git a/pkg/devfile/parser/context/context.go b/pkg/devfile/parser/context/context.go index 54f397c5..a9043cc6 100644 --- a/pkg/devfile/parser/context/context.go +++ b/pkg/devfile/parser/context/context.go @@ -17,6 +17,7 @@ package parser import ( "fmt" + "github.com/devfile/library/v2/pkg/git" "net/url" "os" "path/filepath" @@ -45,16 +46,19 @@ type DevfileCtx struct { // devfile json schema jsonSchema string - //url path of the devfile + // url path of the devfile url string // token is a personal access token used with a private git repo URL token string + // Git is an interface used for git urls + git git.IGitUrl + // filesystem for devfile fs filesystem.Filesystem - // devfile kubernetes components has been coverted from uri to inlined in memory + // devfile kubernetes components has been converted from uri to inlined in memory convertUriToInlined bool } @@ -70,6 +74,7 @@ func NewDevfileCtx(path string) DevfileCtx { func NewURLDevfileCtx(url string) DevfileCtx { return DevfileCtx{ url: url, + git: &git.Url{}, } } @@ -78,6 +83,7 @@ func NewPrivateURLDevfileCtx(url string, token string) DevfileCtx { return DevfileCtx{ url: url, token: token, + git: &git.Url{}, } } @@ -166,6 +172,11 @@ func (d *DevfileCtx) GetToken() string { return d.token } +// GetGit func returns current git interface +func (d *DevfileCtx) GetGit() git.IGitUrl { + return d.git +} + // SetAbsPath sets absolute file path for devfile func (d *DevfileCtx) SetAbsPath() (err error) { // Set devfile absolute path @@ -187,3 +198,8 @@ func (d *DevfileCtx) GetConvertUriToInlined() bool { func (d *DevfileCtx) SetConvertUriToInlined(value bool) { d.convertUriToInlined = value } + +// SetGit sets the git interface +func (d *DevfileCtx) SetGit(git git.IGitUrl) { + d.git = git +} diff --git a/pkg/devfile/parser/context/context_test.go b/pkg/devfile/parser/context/context_test.go index 32097edd..67d8c68f 100644 --- a/pkg/devfile/parser/context/context_test.go +++ b/pkg/devfile/parser/context/context_test.go @@ -16,6 +16,7 @@ package parser import ( + "github.com/devfile/library/v2/pkg/git" "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" @@ -93,11 +94,13 @@ func TestNewURLDevfileCtx(t *testing.T) { d := NewPrivateURLDevfileCtx(url, token) assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL()) assert.Equal(t, "fake-token", d.GetToken()) + assert.Equal(t, &git.Url{}, d.GetGit()) } { d := NewURLDevfileCtx(url) assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL()) assert.Equal(t, "", d.GetToken()) + assert.Equal(t, &git.Url{}, d.GetGit()) } } diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index e7bea9ce..69c5dfae 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/devfile/library/v2/pkg/git" "io/ioutil" "net/url" "os" @@ -141,6 +142,7 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) { context: args.Context, k8sClient: args.K8sClient, httpTimeout: args.HTTPTimeout, + git: d.Ctx.GetGit(), } flattenedDevfile := true @@ -194,6 +196,8 @@ type resolverTools struct { k8sClient client.Client // httpTimeout is the timeout value in seconds passed in from the client. httpTimeout *int + // git is the interface used for git urls + git git.IGitUrl } func populateAndParseDevfile(d DevfileObj, resolveCtx *resolutionContextTree, tool resolverTools, flattenedDevfile bool) (DevfileObj, error) { @@ -444,18 +448,10 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D d.Ctx = devfileCtx.NewURLDevfileCtx(newUri) } - if util.IsGitProviderRepo(newUri) { - gitUrl, err := util.NewGitUrl(newUri) - if err != nil { - return DevfileObj{}, err - } - if gitUrl.IsFile { - destDir := path.Dir(curDevfileCtx.GetAbsPath()) - err = getResourcesFromGit(gitUrl, destDir, tool.httpTimeout, token) - if err != nil { - return DevfileObj{}, err - } - } + destDir := path.Dir(curDevfileCtx.GetAbsPath()) + err = tool.git.DownloadGitRepoResources(newUri, destDir, tool.httpTimeout, token) + if err != nil { + return DevfileObj{}, err } } importReference.Uri = newUri @@ -464,34 +460,6 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D return populateAndParseDevfile(d, newResolveCtx, tool, true) } -func getResourcesFromGit(g *util.GitUrl, destDir string, httpTimeout *int, repoToken string) error { - stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) - if err != nil { - return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) - } - defer os.RemoveAll(stackDir) - - if !g.IsPublic(httpTimeout) { - err = g.SetToken(repoToken, httpTimeout) - if err != nil { - return err - } - } - - err = util.CloneGitRepo(*g, stackDir) - if err != nil { - return err - } - - dir := path.Dir(path.Join(stackDir, g.Path)) - err = util.CopyAllDirFiles(dir, destDir) - if err != nil { - return err - } - - return nil -} - func parseFromRegistry(importReference v1.ImportReference, resolveCtx *resolutionContextTree, tool resolverTools) (d DevfileObj, err error) { id := importReference.Id registryURL := importReference.RegistryUrl @@ -860,6 +828,9 @@ func getKubernetesDefinitionFromUri(uri string, d devfileCtx.DevfileCtx) ([]byte newUri = uri } params := util.HTTPRequestParams{URL: newUri} + if d.GetToken() != "" { + params.Token = d.GetToken() + } data, err = util.DownloadInMemory(params) if err != nil { return nil, errors.Wrapf(err, "error getting kubernetes resources definition information") diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index c2266e00..2d76da1c 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -19,7 +19,7 @@ import ( "context" "fmt" v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" - "github.com/devfile/library/v2/pkg/util" + "github.com/devfile/library/v2/pkg/git" "io/ioutil" "net" "net/http" @@ -2859,7 +2859,7 @@ func Test_parseParentAndPluginFromURI(t *testing.T) { tt.args.devFileObj.Data.AddComponents(plugincomp) } - err := parseParentAndPlugin(tt.args.devFileObj, &resolutionContextTree{}, resolverTools{}) + err := parseParentAndPlugin(tt.args.devFileObj, &resolutionContextTree{}, resolverTools{git: &git.MockGitUrl{}}) // Unexpected error if (err != nil) != (tt.wantErr != nil) { @@ -3069,9 +3069,26 @@ func Test_parseParentAndPlugin_RecursivelyReference(t *testing.T) { testK8sClient := &testingutil.FakeK8sClient{ DevWorkspaceResources: devWorkspaceResources, } + + httpTimeout := 0 + + mockGitUrl := &git.MockGitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "library", + Branch: "main", + Path: "devfile.yaml", + IsFile: true, + } + devFileObj.Ctx.SetGit(mockGitUrl) + devFileObj.Ctx.GetGit().SetToken("valid-token", &httpTimeout) + tool := resolverTools{ - k8sClient: testK8sClient, - context: context.Background(), + k8sClient: testK8sClient, + context: context.Background(), + httpTimeout: &httpTimeout, + git: devFileObj.Ctx.GetGit(), } err := parseParentAndPlugin(devFileObj, &resolutionContextTree{}, tool) @@ -4144,7 +4161,7 @@ func Test_parseFromURI(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseFromURI(tt.importReference, tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{}) + got, err := parseFromURI(tt.importReference, tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{git: &git.MockGitUrl{}}) if (err != nil) != (tt.wantErr != nil) { t.Errorf("Test_parseFromURI() unexpected error: %v, wantErr %v", err, tt.wantErr) } else if err == nil && !reflect.DeepEqual(got.Data, tt.wantDevFile.Data) { @@ -4156,6 +4173,170 @@ func Test_parseFromURI(t *testing.T) { } } +func Test_parseFromURI_GitResources(t *testing.T) { + const ( + invalidToken = "invalid-token" + validToken = "valid-token" + ) + + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + _, err := rw.Write([]byte("schemaVersion: 2.2.0")) + if err != nil { + t.Error(err) + } + })) + + parentDevfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: parent-devfile\nparent:\n uri: \"%s\"", server.URL) + parent := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + _, err := rw.Write([]byte(parentDevfileContent)) + if err != nil { + t.Error(err) + } + })) + + devfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: nested-devfile\nparent:\n uri: \"%s\"", parent.URL) + nestedParent := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + _, err := rw.Write([]byte(devfileContent)) + if err != nil { + t.Error(err) + } + })) + + // Close the server when test finishes + defer server.Close() + defer parent.Close() + defer nestedParent.Close() + + httpTimeoutPrivate := 1 + + publicGitUrl := &git.MockGitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "library", + Branch: "main", + Path: "devfile.yaml", + IsFile: true, + } + + privateBitbucketGitUrl := &git.MockGitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "devfile", + Repo: "registry", + Branch: "main", + Path: "stacks/go/1.0.2/devfile.yaml", + IsFile: true, + } + privateBitbucketGitUrl.SetToken(validToken, &httpTimeoutPrivate) + + privateGitUrl := &git.MockGitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "mock-owner", + Repo: "mock-repo", + Branch: "mock-branch", + Path: "mock/stacks/go/1.0.2/devfile.yaml", + IsFile: true, + } + privateGitUrl.SetToken(validToken, &httpTimeoutPrivate) + + curDevfileContextWithValidToken := devfileCtx.NewPrivateURLDevfileCtx(OutputDevfileYamlPath, validToken) + curDevfileContextWithInvalidToken := devfileCtx.NewPrivateURLDevfileCtx(OutputDevfileYamlPath, invalidToken) + curDevfileContextWithoutToken := devfileCtx.NewURLDevfileCtx(OutputDevfileYamlPath) + + tests := []struct { + name string + curDevfileCtx *devfileCtx.DevfileCtx + gitUrl *git.MockGitUrl + timeout *int + importReference v1.ImportReference + wantError error + }{ + { + name: "private main devfile URL", + curDevfileCtx: &curDevfileContextWithValidToken, + gitUrl: privateGitUrl, + timeout: &httpTimeoutPrivate, + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: server.URL, + }, + }, + wantError: nil, + }, + { + name: "private main devfile Bitbucket URL", + curDevfileCtx: &curDevfileContextWithValidToken, + gitUrl: privateBitbucketGitUrl, + timeout: &httpTimeoutPrivate, + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: server.URL, + }, + }, + wantError: nil, + }, + { + name: "private main devfile with a private parent reference", + curDevfileCtx: &curDevfileContextWithValidToken, + gitUrl: privateGitUrl, + timeout: &httpTimeoutPrivate, + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: parent.URL, + }, + }, + wantError: nil, + }, + { + name: "private main devfile with a private parent with a nested private parent reference", + curDevfileCtx: &curDevfileContextWithValidToken, + gitUrl: privateGitUrl, + timeout: &httpTimeoutPrivate, + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: nestedParent.URL, + }, + }, + wantError: nil, + }, + { + name: "private main devfile without a valid token", + curDevfileCtx: &curDevfileContextWithInvalidToken, + gitUrl: privateGitUrl, + timeout: &httpTimeoutPrivate, + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: server.URL, + }, + }, + wantError: fmt.Errorf("failed to clone repo with token, ensure that the url and token is correct"), + }, + { + name: "public main devfile without a token", + curDevfileCtx: &curDevfileContextWithoutToken, + gitUrl: publicGitUrl, + timeout: &httpTimeoutPrivate, + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: server.URL, + }, + }, + wantError: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := parseFromURI(tt.importReference, *tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{httpTimeout: tt.timeout, git: tt.gitUrl}) + if !reflect.DeepEqual(tt.wantError, err) { + t.Errorf("Expected error: %s, got error: %s", tt.wantError, err) + } + }) + } +} + func Test_parseFromRegistry(t *testing.T) { const ( registry = "127.0.0.1:8080" @@ -4490,7 +4671,7 @@ func Test_parseFromKubeCRD(t *testing.T) { } } -func Test_getResourcesFromGit(t *testing.T) { +func Test_DownloadGitRepoResources(t *testing.T) { destDir, err := ioutil.TempDir("", "") if err != nil { t.Errorf("Failed to create dest dir: %s, error: %v", destDir, err) @@ -4498,49 +4679,40 @@ func Test_getResourcesFromGit(t *testing.T) { defer os.RemoveAll(destDir) httpTimeout := 0 - - invalidGitHubUrl := util.GitUrl{ - Protocol: "", - Host: "", - Owner: "devfile", - Repo: "nonexistent", - Branch: "nonexistent", - } - validGitHubUrl := util.GitUrl{ - Protocol: "https", - Host: "raw.githubusercontent.com", - Owner: "devfile", - Repo: "registry", - Branch: "main", - Path: "stacks/nodejs/devfile.yaml", - IsFile: true, - } + var g git.Url tests := []struct { name string - gitUrl util.GitUrl + url string destDir string + token string wantErr bool }{ { name: "should fail with invalid git url", - gitUrl: invalidGitHubUrl, + url: "non-existent-url", destDir: path.Join(os.TempDir(), "nonexistent"), + token: "", wantErr: true, }, { name: "should be able to get resources from valid git url", - gitUrl: validGitHubUrl, + url: "https://raw.githubusercontent.com/devfile/registry/main/stacks/python/3.0.0/devfile.yaml", destDir: destDir, + token: "", wantErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := getResourcesFromGit(&tt.gitUrl, tt.destDir, &httpTimeout, "") - if (err != nil) != tt.wantErr { - t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err) + gotErr := false + err := g.DownloadGitRepoResources(tt.url, tt.destDir, &httpTimeout, tt.token) + if err != nil { + gotErr = true + } + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("Got error: %t, want error: %t", gotErr, tt.wantErr) } }) } diff --git a/pkg/util/git.go b/pkg/git/git.go similarity index 72% rename from pkg/util/git.go rename to pkg/git/git.go index 30a9edfa..c0f97d92 100644 --- a/pkg/util/git.go +++ b/pkg/git/git.go @@ -13,13 +13,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package util +package git import ( "fmt" + "io/ioutil" "net/url" "os" "os/exec" + "path" "path/filepath" "strings" ) @@ -32,47 +34,48 @@ const ( ) type IGitUrl interface { - ParseGitUrl(fullUrl string) error GitRawFileAPI() string SetToken(token string, httpTimeout *int) error IsPublic(httpTimeout *int) bool + DownloadGitRepoResources(url string, destDir string, httpTimeout *int, token string) error } -type GitUrl struct { +type Url struct { Protocol string // URL scheme Host string // URL domain name Owner string // name of the repo owner Repo string // name of the repo - Branch string // branch name + Branch string // branch name, tag name, revision name, or commit id Path string // path to a directory or file in the repo token string // used for authenticating a private repo IsFile bool // defines if the URL points to a file in the repo } -// NewGitUrl creates a GitUrl from a string url -func NewGitUrl(url string) (*GitUrl, error) { - g := &GitUrl{} - if err := g.ParseGitUrl(url); err != nil { - return g, err +// NewGitUrlWithURL NewGitUrl creates a GitUrl from a string url +func NewGitUrlWithURL(url string) (*Url, error) { + gitUrl, err := ParseGitUrl(url) + if err != nil { + return &gitUrl, err } - return g, nil + return &gitUrl, nil } // ParseGitUrl extracts information from a support git url // Only supports git repositories hosted on GitHub, GitLab, and Bitbucket -func (g *GitUrl) ParseGitUrl(fullUrl string) error { +func ParseGitUrl(fullUrl string) (Url, error) { + var g Url err := ValidateURL(fullUrl) if err != nil { - return err + return g, err } parsedUrl, err := url.Parse(fullUrl) if err != nil { - return err + return g, err } if len(parsedUrl.Path) == 0 { - return fmt.Errorf("url path should not be empty") + return g, fmt.Errorf("url path should not be empty") } if parsedUrl.Host == RawGitHubHost || parsedUrl.Host == GitHubHost { @@ -85,10 +88,103 @@ func (g *GitUrl) ParseGitUrl(fullUrl string) error { err = fmt.Errorf("url host should be a valid GitHub, GitLab, or Bitbucket host; received: %s", parsedUrl.Host) } - return err + return g, err +} + +func (g *Url) GetToken() string { + return g.token +} + +type CommandType string + +const ( + GitCommand CommandType = "git" + unsupportedCmdMsg = "Unsupported command \"%s\" " +) + +// Execute is exposed as a global variable for the purpose of running mock tests +// only "git" is supported +/* #nosec G204 -- used internally to execute various git actions and eventual cleanup of artifacts. Calling methods validate user input to ensure commands are used appropriately */ +var execute = func(baseDir string, cmd CommandType, args ...string) ([]byte, error) { + if cmd == GitCommand { + c := exec.Command(string(cmd), args...) + c.Dir = baseDir + output, err := c.CombinedOutput() + return output, err + } + + return []byte(""), fmt.Errorf(unsupportedCmdMsg, string(cmd)) +} + +func (g *Url) CloneGitRepo(destDir string) error { + exist := CheckPathExists(destDir) + if !exist { + return fmt.Errorf("failed to clone repo, destination directory: '%s' does not exists", destDir) + } + + host := g.Host + if host == RawGitHubHost { + host = GitHubHost + } + + var repoUrl string + if g.GetToken() == "" { + repoUrl = fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, host, g.Owner, g.Repo) + } else { + repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", g.Protocol, g.GetToken(), host, g.Owner, g.Repo) + if g.Host == BitbucketHost { + repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", g.Protocol, g.GetToken(), host, g.Owner, g.Repo) + } + } + + _, err := execute(destDir, "git", "clone", repoUrl, ".") + + if err != nil { + if g.GetToken() == "" { + return fmt.Errorf("failed to clone repo without a token, ensure that a token is set if the repo is private. error: %v", err) + } else { + return fmt.Errorf("failed to clone repo with token, ensure that the url and token is correct. error: %v", err) + } + } + + return nil +} + +func (g *Url) DownloadGitRepoResources(url string, destDir string, httpTimeout *int, token string) error { + gitUrl, err := NewGitUrlWithURL(url) + if err != nil { + return err + } + + if gitUrl.IsGitProviderRepo() && gitUrl.IsFile { + stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) + if err != nil { + return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) + } + defer os.RemoveAll(stackDir) + + if !gitUrl.IsPublic(httpTimeout) { + err = gitUrl.SetToken(token, httpTimeout) + if err != nil { + return err + } + } + + err = gitUrl.CloneGitRepo(stackDir) + if err != nil { + return err + } + + dir := path.Dir(path.Join(stackDir, gitUrl.Path)) + err = CopyAllDirFiles(dir, destDir) + if err != nil { + return err + } + } + return nil } -func (g *GitUrl) parseGitHubUrl(url *url.URL) error { +func (g *Url) parseGitHubUrl(url *url.URL) error { var splitUrl []string var err error @@ -98,6 +194,7 @@ func (g *GitUrl) parseGitHubUrl(url *url.URL) error { if g.Host == RawGitHubHost { g.IsFile = true // raw GitHub urls don't contain "blob" or "tree" + // https://raw.githubusercontent.com/devfile/library/main/devfile.yaml -> [devfile library main devfile.yaml] splitUrl = strings.SplitN(url.Path[1:], "/", 4) if len(splitUrl) == 4 { g.Owner = splitUrl[0] @@ -112,6 +209,7 @@ func (g *GitUrl) parseGitHubUrl(url *url.URL) error { } if g.Host == GitHubHost { + // https://github.com/devfile/library/blob/main/devfile.yaml -> [devfile library blob main devfile.yaml] splitUrl = strings.SplitN(url.Path[1:], "/", 5) if len(splitUrl) < 2 { err = fmt.Errorf("url path should contain /, received: %s", url.Path[1:]) @@ -137,6 +235,8 @@ func (g *GitUrl) parseGitHubUrl(url *url.URL) error { if len(splitUrl) == 5 { g.Branch = splitUrl[3] g.Path = splitUrl[4] + } else if !g.IsFile && len(splitUrl) == 4 { + g.Branch = splitUrl[3] } else { err = fmt.Errorf("url path should contain ////, received: %s", url.Path[1:]) } @@ -146,7 +246,7 @@ func (g *GitUrl) parseGitHubUrl(url *url.URL) error { return err } -func (g *GitUrl) parseGitLabUrl(url *url.URL) error { +func (g *Url) parseGitLabUrl(url *url.URL) error { var splitFile, splitOrg []string var err error @@ -194,7 +294,7 @@ func (g *GitUrl) parseGitLabUrl(url *url.URL) error { return err } -func (g *GitUrl) parseBitbucketUrl(url *url.URL) error { +func (g *Url) parseBitbucketUrl(url *url.URL) error { var splitUrl []string var err error @@ -232,7 +332,7 @@ func (g *GitUrl) parseBitbucketUrl(url *url.URL) error { // SetToken validates the token with a get request to the repo before setting the token // Defaults token to empty on failure. -func (g *GitUrl) SetToken(token string, httpTimeout *int) error { +func (g *Url) SetToken(token string, httpTimeout *int) error { err := g.validateToken(HTTPRequestParams{Token: token, Timeout: httpTimeout}) if err != nil { g.token = "" @@ -244,7 +344,7 @@ func (g *GitUrl) SetToken(token string, httpTimeout *int) error { // IsPublic checks if the GitUrl is public with a get request to the repo using an empty token // Returns true if the request succeeds -func (g *GitUrl) IsPublic(httpTimeout *int) bool { +func (g *Url) IsPublic(httpTimeout *int) bool { err := g.validateToken(HTTPRequestParams{Token: "", Timeout: httpTimeout}) if err != nil { return false @@ -254,7 +354,7 @@ func (g *GitUrl) IsPublic(httpTimeout *int) bool { // validateToken makes a http get request to the repo with the GitUrl token // Returns an error if the get request fails -func (g *GitUrl) validateToken(params HTTPRequestParams) error { +func (g *Url) validateToken(params HTTPRequestParams) error { var apiUrl string switch g.Host { @@ -278,7 +378,7 @@ func (g *GitUrl) validateToken(params HTTPRequestParams) error { } // GitRawFileAPI returns the endpoint for the git providers raw file -func (g *GitUrl) GitRawFileAPI() string { +func (g *Url) GitRawFileAPI() string { var apiRawFile string switch g.Host { @@ -294,52 +394,11 @@ func (g *GitUrl) GitRawFileAPI() string { } // IsGitProviderRepo checks if the url matches a repo from a supported git provider -func IsGitProviderRepo(url string) bool { - if strings.Contains(url, RawGitHubHost) || strings.Contains(url, GitHubHost) || - strings.Contains(url, GitLabHost) || strings.Contains(url, BitbucketHost) { +func (g *Url) IsGitProviderRepo() bool { + switch g.Host { + case GitHubHost, RawGitHubHost, GitLabHost, BitbucketHost: return true + default: + return false } - return false -} - -// CloneGitRepo clones a git repo to a destination directory (either an absolute or relative path) -func CloneGitRepo(g GitUrl, destDir string) error { - exist := CheckPathExists(destDir) - if !exist { - return fmt.Errorf("failed to clone repo, destination directory: '%s' does not exists", destDir) - } - - host := g.Host - if host == RawGitHubHost { - host = GitHubHost - } - - var repoUrl string - if g.token == "" { - repoUrl = fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, host, g.Owner, g.Repo) - } else { - repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) - if g.Host == BitbucketHost { - repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) - } - } - - /* #nosec G204 -- user input is processed into an expected format for the git clone command */ - c := exec.Command("git", "clone", repoUrl, destDir) - c.Dir = destDir - - // set env to skip authentication prompt and directly error out - c.Env = os.Environ() - c.Env = append(c.Env, "GIT_TERMINAL_PROMPT=0", "GIT_ASKPASS=/bin/echo") - - _, err := c.CombinedOutput() - if err != nil { - if g.token == "" { - return fmt.Errorf("failed to clone repo without a token, ensure that a token is set if the repo is private. error: %v", err) - } else { - return fmt.Errorf("failed to clone repo with token, ensure that the url and token is correct. error: %v", err) - } - } - - return nil } diff --git a/pkg/util/git_test.go b/pkg/git/git_test.go similarity index 90% rename from pkg/util/git_test.go rename to pkg/git/git_test.go index d7773fdf..4d614f71 100644 --- a/pkg/util/git_test.go +++ b/pkg/git/git_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package util +package git import ( "github.com/kylelemons/godebug/pretty" @@ -24,11 +24,11 @@ import ( "testing" ) -func Test_NewGitUrl(t *testing.T) { +func Test_ParseGitUrl(t *testing.T) { tests := []struct { name string url string - wantUrl *GitUrl + wantUrl Url wantErr string }{ { @@ -45,7 +45,7 @@ func Test_NewGitUrl(t *testing.T) { { name: "should parse GitHub repo with root path", url: "https://github.com/devfile/library", - wantUrl: &GitUrl{ + wantUrl: Url{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -55,6 +55,32 @@ func Test_NewGitUrl(t *testing.T) { IsFile: false, }, }, + { + name: "should parse GitHub repo with root path and tag", + url: "https://github.com/devfile/library/tree/v2.2.0", + wantUrl: Url{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Branch: "v2.2.0", + Path: "", + IsFile: false, + }, + }, + { + name: "should parse GitHub repo with root path and revision", + url: "https://github.com/devfile/library/tree/0ce592a416fb185564516353891a45016ac7f671", + wantUrl: Url{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Branch: "0ce592a416fb185564516353891a45016ac7f671", + Path: "", + IsFile: false, + }, + }, { name: "should fail with only GitHub host", url: "https://github.com/", @@ -63,7 +89,7 @@ func Test_NewGitUrl(t *testing.T) { { name: "should parse GitHub repo with file path", url: "https://github.com/devfile/library/blob/main/devfile.yaml", - wantUrl: &GitUrl{ + wantUrl: Url{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -76,7 +102,7 @@ func Test_NewGitUrl(t *testing.T) { { name: "should parse GitHub repo with raw file path", url: "https://raw.githubusercontent.com/devfile/library/main/devfile.yaml", - wantUrl: &GitUrl{ + wantUrl: Url{ Protocol: "https", Host: "raw.githubusercontent.com", Owner: "devfile", @@ -120,7 +146,7 @@ func Test_NewGitUrl(t *testing.T) { { name: "should parse GitLab repo with root path", url: "https://gitlab.com/gitlab-org/gitlab-foss", - wantUrl: &GitUrl{ + wantUrl: Url{ Protocol: "https", Host: "gitlab.com", Owner: "gitlab-org", @@ -138,7 +164,7 @@ func Test_NewGitUrl(t *testing.T) { { name: "should parse GitLab repo with file path", url: "https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/README.md", - wantUrl: &GitUrl{ + wantUrl: Url{ Protocol: "https", Host: "gitlab.com", Owner: "gitlab-org", @@ -167,7 +193,7 @@ func Test_NewGitUrl(t *testing.T) { { name: "should parse Bitbucket repo with root path", url: "https://bitbucket.org/fake-owner/fake-public-repo", - wantUrl: &GitUrl{ + wantUrl: Url{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -185,7 +211,7 @@ func Test_NewGitUrl(t *testing.T) { { name: "should parse Bitbucket repo with file path", url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/README.md", - wantUrl: &GitUrl{ + wantUrl: Url{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -198,7 +224,7 @@ func Test_NewGitUrl(t *testing.T) { { name: "should parse Bitbucket file path with nested path", url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/directory/test.txt", - wantUrl: &GitUrl{ + wantUrl: Url{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -211,7 +237,7 @@ func Test_NewGitUrl(t *testing.T) { { name: "should parse Bitbucket repo with raw file path", url: "https://bitbucket.org/fake-owner/fake-public-repo/raw/main/README.md", - wantUrl: &GitUrl{ + wantUrl: Url{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -240,7 +266,7 @@ func Test_NewGitUrl(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := NewGitUrl(tt.url) + got, err := ParseGitUrl(tt.url) if (err != nil) != (tt.wantErr != "") { t.Errorf("Unxpected error: %t, want: %v", err, tt.wantUrl) } else if err == nil && !reflect.DeepEqual(got, tt.wantUrl) { @@ -255,12 +281,12 @@ func Test_NewGitUrl(t *testing.T) { func Test_GetGitRawFileAPI(t *testing.T) { tests := []struct { name string - g GitUrl + g Url want string }{ { name: "Github url", - g: GitUrl{ + g: Url{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -272,7 +298,7 @@ func Test_GetGitRawFileAPI(t *testing.T) { }, { name: "GitLab url", - g: GitUrl{ + g: Url{ Protocol: "https", Host: "gitlab.com", Owner: "gitlab-org", @@ -284,7 +310,7 @@ func Test_GetGitRawFileAPI(t *testing.T) { }, { name: "Bitbucket url", - g: GitUrl{ + g: Url{ Protocol: "https", Host: "bitbucket.org", Owner: "owner", @@ -296,7 +322,7 @@ func Test_GetGitRawFileAPI(t *testing.T) { }, { name: "Empty GitUrl", - g: GitUrl{}, + g: Url{}, want: "", }, } @@ -312,7 +338,7 @@ func Test_GetGitRawFileAPI(t *testing.T) { } func Test_IsPublic(t *testing.T) { - publicGitUrl := GitUrl{ + publicGitUrl := Url{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -321,7 +347,7 @@ func Test_IsPublic(t *testing.T) { token: "fake-token", } - privateGitUrl := GitUrl{ + privateGitUrl := Url{ Protocol: "https", Host: "github.com", Owner: "not", @@ -334,7 +360,7 @@ func Test_IsPublic(t *testing.T) { tests := []struct { name string - g GitUrl + g Url want bool }{ { @@ -365,7 +391,7 @@ func Test_CloneGitRepo(t *testing.T) { tempDirGitLab := t.TempDir() tempDirBitbucket := t.TempDir() - invalidGitUrl := GitUrl{ + invalidGitUrl := Url{ Protocol: "", Host: "", Owner: "nonexistent", @@ -373,7 +399,7 @@ func Test_CloneGitRepo(t *testing.T) { Branch: "nonexistent", } - validPublicGitHubUrl := GitUrl{ + validPublicGitHubUrl := Url{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -381,7 +407,7 @@ func Test_CloneGitRepo(t *testing.T) { Branch: "main", } - validPublicGitLabUrl := GitUrl{ + validPublicGitLabUrl := Url{ Protocol: "https", Host: "gitlab.com", Owner: "mike-hoang", @@ -389,7 +415,7 @@ func Test_CloneGitRepo(t *testing.T) { Branch: "main", } - validPublicBitbucketUrl := GitUrl{ + validPublicBitbucketUrl := Url{ Protocol: "https", Host: "bitbucket.org", Owner: "mike-hoang", @@ -397,7 +423,7 @@ func Test_CloneGitRepo(t *testing.T) { Branch: "master", } - invalidPrivateGitHubRepo := GitUrl{ + invalidPrivateGitHubRepo := Url{ Protocol: "https", Host: "github.com", Owner: "fake-owner", @@ -412,7 +438,7 @@ func Test_CloneGitRepo(t *testing.T) { tests := []struct { name string - gitUrl GitUrl + gitUrl Url destDir string wantErr string }{ @@ -453,7 +479,7 @@ func Test_CloneGitRepo(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := CloneGitRepo(tt.gitUrl, tt.destDir) + err := tt.gitUrl.CloneGitRepo(tt.destDir) if (err != nil) != (tt.wantErr != "") { t.Errorf("Unxpected error: %t, want: %v", err, tt.wantErr) } else if err != nil { diff --git a/pkg/git/mock.go b/pkg/git/mock.go new file mode 100644 index 00000000..1351dbb9 --- /dev/null +++ b/pkg/git/mock.go @@ -0,0 +1,136 @@ +// +// Copyright 2023 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + "fmt" + "io/ioutil" + "net/url" + "os" +) + +type MockGitUrl struct { + Protocol string // URL scheme + Host string // URL domain name + Owner string // name of the repo owner + Repo string // name of the repo + Branch string // branch name + Path string // path to a directory or file in the repo + token string // used for authenticating a private repo + IsFile bool // defines if the URL points to a file in the repo +} + +func (m *MockGitUrl) GetToken() string { + return m.token +} + +var mockExecute = func(baseDir string, cmd CommandType, args ...string) ([]byte, error) { + if cmd == GitCommand { + u, _ := url.Parse(args[1]) + password, hasPassword := u.User.Password() + + if hasPassword { + switch password { + case "valid-token": + return []byte("test"), nil + default: + return []byte(""), fmt.Errorf("not a valid token") + } + } + + return []byte("test"), nil + } + + return []byte(""), fmt.Errorf(unsupportedCmdMsg, string(cmd)) +} + +func (m *MockGitUrl) CloneGitRepo(destDir string) error { + exist := CheckPathExists(destDir) + if !exist { + return fmt.Errorf("failed to clone repo, destination directory: '%s' does not exists", destDir) + } + + host := m.Host + if host == RawGitHubHost { + host = GitHubHost + } + + var repoUrl string + if m.GetToken() == "" { + repoUrl = fmt.Sprintf("%s://%s/%s/%s.git", m.Protocol, host, m.Owner, m.Repo) + } else { + repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", m.Protocol, m.GetToken(), host, m.Owner, m.Repo) + if m.Host == BitbucketHost { + repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", m.Protocol, m.GetToken(), host, m.Owner, m.Repo) + } + } + + _, err := mockExecute(destDir, "git", "clone", repoUrl, ".") + + if err != nil { + if m.GetToken() == "" { + return fmt.Errorf("failed to clone repo without a token, ensure that a token is set if the repo is private") + } else { + return fmt.Errorf("failed to clone repo with token, ensure that the url and token is correct") + } + } + + return nil +} + +func (m *MockGitUrl) DownloadGitRepoResources(url string, destDir string, httpTimeout *int, token string) error { + gitUrl := m + if gitUrl.IsGitProviderRepo() && gitUrl.IsFile { + stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) + if err != nil { + return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) + } + defer os.RemoveAll(stackDir) + + if !gitUrl.IsPublic(httpTimeout) { + err = m.SetToken(token, httpTimeout) + if err != nil { + return err + } + } + + err = gitUrl.CloneGitRepo(stackDir) + if err != nil { + return err + } + } + return nil +} + +func (m *MockGitUrl) SetToken(token string, httpTimeout *int) error { + m.token = token + return nil +} + +func (m *MockGitUrl) IsPublic(httpTimeout *int) bool { + if *httpTimeout != 0 { + return false + } + return true +} + +func (m *MockGitUrl) GitRawFileAPI() string { + return "" +} + +func (m *MockGitUrl) IsGitProviderRepo() bool { + return true +} diff --git a/pkg/git/util.go b/pkg/git/util.go new file mode 100644 index 00000000..79ed650a --- /dev/null +++ b/pkg/git/util.go @@ -0,0 +1,260 @@ +// +// Copyright 2023 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + "fmt" + "github.com/devfile/library/v2/pkg/testingutil/filesystem" + "github.com/gregjones/httpcache" + "github.com/gregjones/httpcache/diskcache" + "github.com/pkg/errors" + "io" + "io/ioutil" + "k8s.io/klog" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "time" +) + +const ( + HTTPRequestResponseTimeout = 30 * time.Second // HTTPRequestTimeout configures timeout of all HTTP requests +) + +// httpCacheDir determines directory where odo will cache HTTP responses +var httpCacheDir = filepath.Join(os.TempDir(), "odohttpcache") + +// HTTPRequestParams holds parameters of forming http request +type HTTPRequestParams struct { + URL string + Token string + Timeout *int + TelemetryClientName string //optional client name for telemetry +} + +// HTTPGetRequest gets resource contents given URL and token (if applicable) +// cacheFor determines how long the response should be cached (in minutes), 0 for no caching +func HTTPGetRequest(request HTTPRequestParams, cacheFor int) ([]byte, error) { + // Build http request + req, err := http.NewRequest("GET", request.URL, nil) + if err != nil { + return nil, err + } + if request.Token != "" { + bearer := "Bearer " + request.Token + req.Header.Add("Authorization", bearer) + } + + //add the telemetry client name + req.Header.Add("Client", request.TelemetryClientName) + + overriddenTimeout := HTTPRequestResponseTimeout + timeout := request.Timeout + if timeout != nil { + //if value is invalid, the default will be used + if *timeout > 0 { + //convert timeout to seconds + overriddenTimeout = time.Duration(*timeout) * time.Second + klog.V(4).Infof("HTTP request and response timeout overridden value is %v ", overriddenTimeout) + } else { + klog.V(4).Infof("Invalid httpTimeout is passed in, using default value") + } + + } + + httpClient := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ResponseHeaderTimeout: overriddenTimeout, + }, + Timeout: overriddenTimeout, + } + + klog.V(4).Infof("HTTPGetRequest: %s", req.URL.String()) + + if cacheFor > 0 { + // if there is an error during cache setup we show warning and continue without using cache + cacheError := false + httpCacheTime := time.Duration(cacheFor) * time.Minute + + // make sure that cache directory exists + err = os.MkdirAll(httpCacheDir, 0750) + if err != nil { + cacheError = true + klog.WarningDepth(4, "Unable to setup cache: ", err) + } + err = cleanHttpCache(httpCacheDir, httpCacheTime) + if err != nil { + cacheError = true + klog.WarningDepth(4, "Unable to clean up cache directory: ", err) + } + + if !cacheError { + httpClient.Transport = httpcache.NewTransport(diskcache.New(httpCacheDir)) + klog.V(4).Infof("Response will be cached in %s for %s", httpCacheDir, httpCacheTime) + } else { + klog.V(4).Info("Response won't be cached.") + } + } + + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.Header.Get(httpcache.XFromCache) != "" { + klog.V(4).Infof("Cached response used.") + } + + // We have a non 1xx / 2xx status, return an error + if (resp.StatusCode - 300) > 0 { + return nil, errors.Errorf("failed to retrieve %s, %v: %s", request.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + // Process http response + bytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return bytes, err +} + +// ValidateURL validates the URL +func ValidateURL(sourceURL string) error { + u, err := url.Parse(sourceURL) + if err != nil { + return err + } + + if len(u.Host) == 0 || len(u.Scheme) == 0 { + return errors.New("URL is invalid") + } + + return nil +} + +// cleanHttpCache checks cacheDir and deletes all files that were modified more than cacheTime back +func cleanHttpCache(cacheDir string, cacheTime time.Duration) error { + cacheFiles, err := ioutil.ReadDir(cacheDir) + if err != nil { + return err + } + + for _, f := range cacheFiles { + if f.ModTime().Add(cacheTime).Before(time.Now()) { + klog.V(4).Infof("Removing cache file %s, because it is older than %s", f.Name(), cacheTime.String()) + err := os.Remove(filepath.Join(cacheDir, f.Name())) + if err != nil { + return err + } + } + } + return nil +} + +// CheckPathExists checks if a path exists or not +func CheckPathExists(path string) bool { + return checkPathExistsOnFS(path, filesystem.DefaultFs{}) +} + +func checkPathExistsOnFS(path string, fs filesystem.Filesystem) bool { + if _, err := fs.Stat(path); !os.IsNotExist(err) { + // path to file does exist + return true + } + klog.V(4).Infof("path %s doesn't exist, skipping it", path) + return false +} + +// CopyAllDirFiles recursively copies a source directory to a destination directory +func CopyAllDirFiles(srcDir, destDir string) error { + return copyAllDirFilesOnFS(srcDir, destDir, filesystem.DefaultFs{}) +} + +func copyAllDirFilesOnFS(srcDir, destDir string, fs filesystem.Filesystem) error { + var info os.FileInfo + + files, err := fs.ReadDir(srcDir) + if err != nil { + return errors.Wrapf(err, "failed reading dir %v", srcDir) + } + + for _, file := range files { + srcPath := path.Join(srcDir, file.Name()) + destPath := path.Join(destDir, file.Name()) + + if file.IsDir() { + if info, err = fs.Stat(srcPath); err != nil { + return err + } + if err = fs.MkdirAll(destPath, info.Mode()); err != nil { + return err + } + if err = copyAllDirFilesOnFS(srcPath, destPath, fs); err != nil { + return err + } + } else { + if file.Name() == "devfile.yaml" { + continue + } + // Only copy files that do not exist in the destination directory + if !checkPathExistsOnFS(destPath, fs) { + if err := copyFileOnFs(srcPath, destPath, fs); err != nil { + return errors.Wrapf(err, "failed to copy %s to %s", srcPath, destPath) + } + } + } + } + return nil +} + +// copied from: https://github.com/devfile/registry-support/blob/main/index/generator/library/util.go +func copyFileOnFs(src, dst string, fs filesystem.Filesystem) error { + var err error + var srcinfo os.FileInfo + + srcfd, err := fs.Open(src) + if err != nil { + return err + } + defer func() { + if e := srcfd.Close(); e != nil { + fmt.Printf("err occurred while closing file: %v", e) + } + }() + + dstfd, err := fs.Create(dst) + if err != nil { + return err + } + defer func() { + if e := dstfd.Close(); e != nil { + fmt.Printf("err occurred while closing file: %v", e) + } + }() + + if _, err = io.Copy(dstfd, srcfd); err != nil { + return err + } + if srcinfo, err = fs.Stat(src); err != nil { + return err + } + return fs.Chmod(dst, srcinfo.Mode()) +} diff --git a/pkg/git/util_test.go b/pkg/git/util_test.go new file mode 100644 index 00000000..3fd3126e --- /dev/null +++ b/pkg/git/util_test.go @@ -0,0 +1,122 @@ +// +// Copyright 2021-2022 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + "github.com/devfile/library/v2/pkg/testingutil/filesystem" + "net/http" + "net/http/httptest" + "reflect" + "testing" +) + +func TestHTTPGetRequest(t *testing.T) { + invalidHTTPTimeout := -1 + validHTTPTimeout := 20 + + // Start a local HTTP server + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + // Send response to be tested + _, err := rw.Write([]byte("OK")) + if err != nil { + t.Error(err) + } + })) + // Close the server when test finishes + defer server.Close() + + tests := []struct { + name string + url string + want []byte + timeout *int + }{ + { + name: "Case 1: Input url is valid", + url: server.URL, + // Want(Expected) result is "OK" + // According to Unicode table: O == 79, K == 75 + want: []byte{79, 75}, + }, + { + name: "Case 2: Input url is invalid", + url: "invalid", + want: nil, + }, + { + name: "Case 3: Test invalid httpTimeout, default timeout will be used", + url: server.URL, + timeout: &invalidHTTPTimeout, + want: []byte{79, 75}, + }, + { + name: "Case 4: Test valid httpTimeout", + url: server.URL, + timeout: &validHTTPTimeout, + want: []byte{79, 75}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + request := HTTPRequestParams{ + URL: tt.url, + Timeout: tt.timeout, + } + got, err := HTTPGetRequest(request, 0) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Got: %v, want: %v", got, tt.want) + t.Logf("Error message is: %v", err) + } + }) + } +} + +func TestCheckPathExists(t *testing.T) { + fs := filesystem.NewFakeFs() + fs.MkdirAll("/path/to/devfile", 0755) + fs.WriteFile("/path/to/devfile/devfile.yaml", []byte(""), 0755) + + file := "/path/to/devfile/devfile.yaml" + missingFile := "/path/to/not/devfile" + + tests := []struct { + name string + filePath string + want bool + }{ + { + name: "should be able to get file that exists", + filePath: file, + want: true, + }, + { + name: "should fail if file does not exist", + filePath: missingFile, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := checkPathExistsOnFS(tt.filePath, fs) + if !reflect.DeepEqual(result, tt.want) { + t.Errorf("Got error: %t, want error: %t", result, tt.want) + } + }) + } +} diff --git a/pkg/util/mock.go b/pkg/util/mock.go deleted file mode 100644 index 061986cb..00000000 --- a/pkg/util/mock.go +++ /dev/null @@ -1,57 +0,0 @@ -// -// Copyright 2023 Red Hat, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import "net/http" - -var ( - GetDoFunc func(req *http.Request) (*http.Response, error) - GetParseGitUrlFunc func(url string) error - GetGitRawFileAPIFunc func() string - GetSetTokenFunc func(token string, httpTimeout *int) error - GetIsPublicFunc func(httpTimeout *int) bool -) - -type MockClient struct { - DoFunc func(req *http.Request) (*http.Response, error) -} - -func (m *MockClient) Do(req *http.Request) (*http.Response, error) { - return GetDoFunc(req) -} - -type MockGitUrl struct { - ParseGitUrlFunc func(fullUrl string) error - GetGitRawFileAPIFunc func(url string) string - SetTokenFunc func(token string, httpTimeout *int) error - IsPublicFunc func(httpTimeout *int) bool -} - -func (m *MockGitUrl) ParseGitUrl(fullUrl string) error { - return GetParseGitUrlFunc(fullUrl) -} - -func (m *MockGitUrl) GitRawFileAPI() string { - return GetGitRawFileAPIFunc() -} - -func (m *MockGitUrl) SetToken(token string, httpTimeout *int) error { - return GetSetTokenFunc(token, httpTimeout) -} - -func (m *MockGitUrl) IsPublic(httpTimeout *int) bool { - return GetIsPublicFunc(httpTimeout) -} diff --git a/pkg/util/util.go b/pkg/util/util.go index 432b92f0..6db76a8a 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -21,6 +21,7 @@ import ( "bytes" "crypto/rand" "fmt" + "github.com/devfile/library/v2/pkg/git" "io" "io/ioutil" "math/big" @@ -882,6 +883,15 @@ func ConvertGitSSHRemoteToHTTPS(remote string) string { return remote } +// IsGitProviderRepo checks if the url matches a repo from a supported git provider +func IsGitProviderRepo(url string) bool { + if strings.Contains(url, git.RawGitHubHost) || strings.Contains(url, git.GitHubHost) || + strings.Contains(url, git.GitLabHost) || strings.Contains(url, git.BitbucketHost) { + return true + } + return false +} + // GetAndExtractZip downloads a zip file from a URL with a http prefix or // takes an absolute path prefixed with file:// and extracts it to a destination. // pathToUnzip specifies the path within the zip folder to extract @@ -1088,20 +1098,20 @@ func DownloadInMemory(params HTTPRequestParams) ([]byte, error) { ResponseHeaderTimeout: HTTPRequestResponseTimeout, }, Timeout: HTTPRequestResponseTimeout} + var g git.IGitUrl var err error - var gitUrl = &GitUrl{} if IsGitProviderRepo(params.URL) { - gitUrl, err = NewGitUrl(params.URL) + g, err = git.NewGitUrlWithURL(params.URL) if err != nil { return nil, errors.Errorf("failed to parse git repo. error: %v", err) } } - return downloadInMemoryWithClient(params, httpClient, gitUrl) + return downloadInMemoryWithClient(params, httpClient, g) } -func downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient, g IGitUrl) ([]byte, error) { +func downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient, g git.IGitUrl) ([]byte, error) { var url string url = params.URL req, err := http.NewRequest("GET", url, nil) diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index c89466c6..fb01e4fc 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -16,7 +16,6 @@ package util import ( - "bytes" "fmt" "github.com/devfile/library/v2/pkg/testingutil/filesystem" "github.com/kylelemons/godebug/pretty" @@ -939,79 +938,6 @@ func TestDownloadFile(t *testing.T) { } } -func TestDownloadInMemory_GitRepo(t *testing.T) { - respBody := []byte("test response body") - resp := &http.Response{ - StatusCode: http.StatusOK, - Body: ioutil.NopCloser(bytes.NewReader(respBody)), - } - - var Client = &MockClient{} - GetDoFunc = func(req *http.Request) (*http.Response, error) { - if req.Header.Get("Authorization") == "" { - return nil, fmt.Errorf("missing authorization header") - } - return resp, nil - } - - var GitUrlMock = &MockGitUrl{} - GetGitRawFileAPIFunc = func() string { - return "" - } - GetSetTokenFunc = func(token string, httpTimeout *int) error { - return nil - } - - tests := []struct { - name string - params HTTPRequestParams - GetIsPublicFunc func(httpTimeout *int) bool - want []byte - wantErr string - }{ - { - name: "Case 1: Private Github repo with token", - params: HTTPRequestParams{ - URL: "https://github.com/myorg/myrepo/file.txt", - Token: "fake-token", - }, - GetIsPublicFunc: func(httpTimeout *int) bool { return false }, - want: []byte("test response body"), - wantErr: "", - }, - { - name: "Case 2: Private Github repo without token", - params: HTTPRequestParams{ - URL: "https://github.com/myorg/myrepo/file.txt", - }, - GetIsPublicFunc: func(httpTimeout *int) bool { return true }, - wantErr: "missing authorization header", - }, - { - name: "Case 3: Non git provider repo", - params: HTTPRequestParams{ - URL: "https://repo.com/myorg/myrepo/file.txt", - Token: "", - }, - wantErr: "missing authorization header", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - GetIsPublicFunc = tt.GetIsPublicFunc - result, err := downloadInMemoryWithClient(tt.params, Client, GitUrlMock) - if (err != nil) != (tt.wantErr != "") { - t.Errorf("Unxpected error: %t, want: %v", err, tt.want) - } else if err == nil && !reflect.DeepEqual(result, tt.want) { - t.Errorf("Expected: %v, received: %v, difference at %v", tt.want, result, pretty.Compare(tt.want, result)) - } else if err != nil { - assert.Regexp(t, tt.wantErr, err.Error(), "Error message should match") - } - }) - } -} - func TestDownloadInMemory(t *testing.T) { // Start a local HTTP server server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { From 730623677d9e3c5aa876883a2cd01074260fc5be Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Tue, 18 Apr 2023 15:53:34 -0400 Subject: [PATCH 10/15] switch to revision after cloning repo Signed-off-by: Michael Hoang --- pkg/devfile/parser/parse_test.go | 49 +++++++++++++++++++++++++------- pkg/git/git.go | 25 ++++++++++------ pkg/git/git_test.go | 42 +++++++++++++-------------- pkg/git/mock.go | 39 ++++++++++++++++++------- 4 files changed, 104 insertions(+), 51 deletions(-) diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index 2d76da1c..ee104a86 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -3077,7 +3077,7 @@ func Test_parseParentAndPlugin_RecursivelyReference(t *testing.T) { Host: "raw.githubusercontent.com", Owner: "devfile", Repo: "library", - Branch: "main", + Revision: "main", Path: "devfile.yaml", IsFile: true, } @@ -4175,8 +4175,9 @@ func Test_parseFromURI(t *testing.T) { func Test_parseFromURI_GitResources(t *testing.T) { const ( - invalidToken = "invalid-token" - validToken = "valid-token" + invalidToken = "invalid-token" + validToken = "valid-token" + invalidRevision = "invalid-revision" ) server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { @@ -4214,17 +4215,28 @@ func Test_parseFromURI_GitResources(t *testing.T) { Host: "raw.githubusercontent.com", Owner: "devfile", Repo: "library", - Branch: "main", + Revision: "main", Path: "devfile.yaml", IsFile: true, } + privateGitUrlInvalidRevision := &git.MockGitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "library", + Revision: invalidRevision, + Path: "devfile.yaml", + IsFile: true, + } + privateGitUrlInvalidRevision.SetToken(validToken, &httpTimeoutPrivate) + privateBitbucketGitUrl := &git.MockGitUrl{ Protocol: "https", Host: "bitbucket.org", Owner: "devfile", Repo: "registry", - Branch: "main", + Revision: "main", Path: "stacks/go/1.0.2/devfile.yaml", IsFile: true, } @@ -4235,7 +4247,7 @@ func Test_parseFromURI_GitResources(t *testing.T) { Host: "github.com", Owner: "mock-owner", Repo: "mock-repo", - Branch: "mock-branch", + Revision: "mock-branch", Path: "mock/stacks/go/1.0.2/devfile.yaml", IsFile: true, } @@ -4245,13 +4257,16 @@ func Test_parseFromURI_GitResources(t *testing.T) { curDevfileContextWithInvalidToken := devfileCtx.NewPrivateURLDevfileCtx(OutputDevfileYamlPath, invalidToken) curDevfileContextWithoutToken := devfileCtx.NewURLDevfileCtx(OutputDevfileYamlPath) + invalidTokenError := "failed to clone repo with token, ensure that the url and token is correct" + invalidGitSwitchError := "failed to switch repo to revision*" + tests := []struct { name string curDevfileCtx *devfileCtx.DevfileCtx gitUrl *git.MockGitUrl timeout *int importReference v1.ImportReference - wantError error + wantError *string }{ { name: "private main devfile URL", @@ -4311,7 +4326,7 @@ func Test_parseFromURI_GitResources(t *testing.T) { Uri: server.URL, }, }, - wantError: fmt.Errorf("failed to clone repo with token, ensure that the url and token is correct"), + wantError: &invalidTokenError, }, { name: "public main devfile without a token", @@ -4325,13 +4340,27 @@ func Test_parseFromURI_GitResources(t *testing.T) { }, wantError: nil, }, + { + name: "private parent devfile with invalid revision", + curDevfileCtx: &curDevfileContextWithoutToken, + gitUrl: privateGitUrlInvalidRevision, + timeout: &httpTimeoutPrivate, + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: parent.URL, + }, + }, + wantError: &invalidGitSwitchError, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { _, err := parseFromURI(tt.importReference, *tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{httpTimeout: tt.timeout, git: tt.gitUrl}) - if !reflect.DeepEqual(tt.wantError, err) { - t.Errorf("Expected error: %s, got error: %s", tt.wantError, err) + if (err != nil) != (tt.wantError != nil) { + t.Errorf("Unexpected error: %v, wantErr %v", err, tt.wantError) + } else if tt.wantError != nil { + assert.Regexp(t, *tt.wantError, err.Error(), "Error message should match") } }) } diff --git a/pkg/git/git.go b/pkg/git/git.go index c0f97d92..5c1a751e 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -45,7 +45,7 @@ type Url struct { Host string // URL domain name Owner string // name of the repo owner Repo string // name of the repo - Branch string // branch name, tag name, revision name, or commit id + Revision string // branch name, tag name, or commit id Path string // path to a directory or file in the repo token string // used for authenticating a private repo IsFile bool // defines if the URL points to a file in the repo @@ -137,7 +137,7 @@ func (g *Url) CloneGitRepo(destDir string) error { } } - _, err := execute(destDir, "git", "clone", repoUrl, ".") + _, err := execute(destDir, "git", "clone", repoUrl, destDir) if err != nil { if g.GetToken() == "" { @@ -147,6 +147,13 @@ func (g *Url) CloneGitRepo(destDir string) error { } } + if g.Revision != "" { + _, err := execute(destDir, "git", "switch", "--detach", "origin/"+g.Revision) + if err != nil { + return fmt.Errorf("failed to switch repo to revision. repo dir: %v, revision: %v", destDir, g.Revision) + } + } + return nil } @@ -199,7 +206,7 @@ func (g *Url) parseGitHubUrl(url *url.URL) error { if len(splitUrl) == 4 { g.Owner = splitUrl[0] g.Repo = splitUrl[1] - g.Branch = splitUrl[2] + g.Revision = splitUrl[2] g.Path = splitUrl[3] } else { // raw GitHub urls have to be a file @@ -233,10 +240,10 @@ func (g *Url) parseGitHubUrl(url *url.URL) error { // url has a path to a file or directory if len(splitUrl) == 5 { - g.Branch = splitUrl[3] + g.Revision = splitUrl[3] g.Path = splitUrl[4] } else if !g.IsFile && len(splitUrl) == 4 { - g.Branch = splitUrl[3] + g.Revision = splitUrl[3] } else { err = fmt.Errorf("url path should contain ////, received: %s", url.Path[1:]) } @@ -278,7 +285,7 @@ func (g *Url) parseGitLabUrl(url *url.URL) error { if len(splitFile) == 3 { if splitFile[0] == "blob" || splitFile[0] == "tree" || splitFile[0] == "raw" { - g.Branch = splitFile[1] + g.Revision = splitFile[1] g.Path = splitFile[2] ext := filepath.Ext(g.Path) if ext != "" { @@ -313,7 +320,7 @@ func (g *Url) parseBitbucketUrl(url *url.URL) error { g.Repo = splitUrl[1] if len(splitUrl) == 5 { if splitUrl[2] == "raw" || splitUrl[2] == "src" { - g.Branch = splitUrl[3] + g.Revision = splitUrl[3] g.Path = splitUrl[4] ext := filepath.Ext(g.Path) if ext != "" { @@ -383,11 +390,11 @@ func (g *Url) GitRawFileAPI() string { switch g.Host { case GitHubHost, RawGitHubHost: - apiRawFile = fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", g.Owner, g.Repo, g.Branch, g.Path) + apiRawFile = fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", g.Owner, g.Repo, g.Revision, g.Path) case GitLabHost: apiRawFile = fmt.Sprintf("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/files/%s/raw", g.Owner, g.Repo, g.Path) case BitbucketHost: - apiRawFile = fmt.Sprintf("https://api.bitbucket.org/2.0/repositories/%s/%s/src/%s/%s", g.Owner, g.Repo, g.Branch, g.Path) + apiRawFile = fmt.Sprintf("https://api.bitbucket.org/2.0/repositories/%s/%s/src/%s/%s", g.Owner, g.Repo, g.Revision, g.Path) } return apiRawFile diff --git a/pkg/git/git_test.go b/pkg/git/git_test.go index 4d614f71..a958185d 100644 --- a/pkg/git/git_test.go +++ b/pkg/git/git_test.go @@ -50,7 +50,7 @@ func Test_ParseGitUrl(t *testing.T) { Host: "github.com", Owner: "devfile", Repo: "library", - Branch: "", + Revision: "", Path: "", IsFile: false, }, @@ -63,7 +63,7 @@ func Test_ParseGitUrl(t *testing.T) { Host: "github.com", Owner: "devfile", Repo: "library", - Branch: "v2.2.0", + Revision: "v2.2.0", Path: "", IsFile: false, }, @@ -76,7 +76,7 @@ func Test_ParseGitUrl(t *testing.T) { Host: "github.com", Owner: "devfile", Repo: "library", - Branch: "0ce592a416fb185564516353891a45016ac7f671", + Revision: "0ce592a416fb185564516353891a45016ac7f671", Path: "", IsFile: false, }, @@ -94,7 +94,7 @@ func Test_ParseGitUrl(t *testing.T) { Host: "github.com", Owner: "devfile", Repo: "library", - Branch: "main", + Revision: "main", Path: "devfile.yaml", IsFile: true, }, @@ -107,7 +107,7 @@ func Test_ParseGitUrl(t *testing.T) { Host: "raw.githubusercontent.com", Owner: "devfile", Repo: "library", - Branch: "main", + Revision: "main", Path: "devfile.yaml", IsFile: true, }, @@ -151,7 +151,7 @@ func Test_ParseGitUrl(t *testing.T) { Host: "gitlab.com", Owner: "gitlab-org", Repo: "gitlab-foss", - Branch: "", + Revision: "", Path: "", IsFile: false, }, @@ -169,7 +169,7 @@ func Test_ParseGitUrl(t *testing.T) { Host: "gitlab.com", Owner: "gitlab-org", Repo: "gitlab-foss", - Branch: "master", + Revision: "master", Path: "README.md", IsFile: true, }, @@ -198,7 +198,7 @@ func Test_ParseGitUrl(t *testing.T) { Host: "bitbucket.org", Owner: "fake-owner", Repo: "fake-public-repo", - Branch: "", + Revision: "", Path: "", IsFile: false, }, @@ -216,7 +216,7 @@ func Test_ParseGitUrl(t *testing.T) { Host: "bitbucket.org", Owner: "fake-owner", Repo: "fake-public-repo", - Branch: "main", + Revision: "main", Path: "README.md", IsFile: true, }, @@ -229,7 +229,7 @@ func Test_ParseGitUrl(t *testing.T) { Host: "bitbucket.org", Owner: "fake-owner", Repo: "fake-public-repo", - Branch: "main", + Revision: "main", Path: "directory/test.txt", IsFile: true, }, @@ -242,7 +242,7 @@ func Test_ParseGitUrl(t *testing.T) { Host: "bitbucket.org", Owner: "fake-owner", Repo: "fake-public-repo", - Branch: "main", + Revision: "main", Path: "README.md", IsFile: true, }, @@ -291,7 +291,7 @@ func Test_GetGitRawFileAPI(t *testing.T) { Host: "github.com", Owner: "devfile", Repo: "library", - Branch: "main", + Revision: "main", Path: "tests/README.md", }, want: "https://raw.githubusercontent.com/devfile/library/main/tests/README.md", @@ -303,7 +303,7 @@ func Test_GetGitRawFileAPI(t *testing.T) { Host: "gitlab.com", Owner: "gitlab-org", Repo: "gitlab", - Branch: "master", + Revision: "master", Path: "README.md", }, want: "https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab/repository/files/README.md/raw", @@ -315,7 +315,7 @@ func Test_GetGitRawFileAPI(t *testing.T) { Host: "bitbucket.org", Owner: "owner", Repo: "repo-name", - Branch: "main", + Revision: "main", Path: "path/to/file.md", }, want: "https://api.bitbucket.org/2.0/repositories/owner/repo-name/src/main/path/to/file.md", @@ -343,7 +343,7 @@ func Test_IsPublic(t *testing.T) { Host: "github.com", Owner: "devfile", Repo: "library", - Branch: "main", + Revision: "main", token: "fake-token", } @@ -352,7 +352,7 @@ func Test_IsPublic(t *testing.T) { Host: "github.com", Owner: "not", Repo: "a-valid", - Branch: "none", + Revision: "none", token: "fake-token", } @@ -396,7 +396,7 @@ func Test_CloneGitRepo(t *testing.T) { Host: "", Owner: "nonexistent", Repo: "nonexistent", - Branch: "nonexistent", + Revision: "nonexistent", } validPublicGitHubUrl := Url{ @@ -404,7 +404,7 @@ func Test_CloneGitRepo(t *testing.T) { Host: "github.com", Owner: "devfile", Repo: "library", - Branch: "main", + Revision: "main", } validPublicGitLabUrl := Url{ @@ -412,7 +412,7 @@ func Test_CloneGitRepo(t *testing.T) { Host: "gitlab.com", Owner: "mike-hoang", Repo: "public-testing-repo", - Branch: "main", + Revision: "main", } validPublicBitbucketUrl := Url{ @@ -420,7 +420,7 @@ func Test_CloneGitRepo(t *testing.T) { Host: "bitbucket.org", Owner: "mike-hoang", Repo: "public-testing-repo", - Branch: "master", + Revision: "master", } invalidPrivateGitHubRepo := Url{ @@ -428,7 +428,7 @@ func Test_CloneGitRepo(t *testing.T) { Host: "github.com", Owner: "fake-owner", Repo: "fake-private-repo", - Branch: "master", + Revision: "master", token: "fake-github-token", } diff --git a/pkg/git/mock.go b/pkg/git/mock.go index 1351dbb9..a6dca5b7 100644 --- a/pkg/git/mock.go +++ b/pkg/git/mock.go @@ -20,6 +20,7 @@ import ( "io/ioutil" "net/url" "os" + "strings" ) type MockGitUrl struct { @@ -27,7 +28,7 @@ type MockGitUrl struct { Host string // URL domain name Owner string // name of the repo owner Repo string // name of the repo - Branch string // branch name + Revision string // branch name, tag name, or commit id Path string // path to a directory or file in the repo token string // used for authenticating a private repo IsFile bool // defines if the URL points to a file in the repo @@ -39,19 +40,28 @@ func (m *MockGitUrl) GetToken() string { var mockExecute = func(baseDir string, cmd CommandType, args ...string) ([]byte, error) { if cmd == GitCommand { - u, _ := url.Parse(args[1]) - password, hasPassword := u.User.Password() - - if hasPassword { - switch password { - case "valid-token": - return []byte("test"), nil - default: - return []byte(""), fmt.Errorf("not a valid token") + if len(args) > 0 && args[0] == "clone" { + u, _ := url.Parse(args[1]) + password, hasPassword := u.User.Password() + + if hasPassword { + switch password { + case "valid-token": + return []byte("test"), nil + default: + return []byte(""), fmt.Errorf("not a valid token") + } } + return []byte("test"), nil } - return []byte("test"), nil + if len(args) > 0 && args[0] == "switch" { + revision := strings.TrimPrefix(args[2], "origin/") + if revision != "invalid-revision" { + return []byte("git switched to revision"), nil + } + return []byte(""), fmt.Errorf("failed to switch revision") + } } return []byte(""), fmt.Errorf(unsupportedCmdMsg, string(cmd)) @@ -88,6 +98,13 @@ func (m *MockGitUrl) CloneGitRepo(destDir string) error { } } + if m.Revision != "" { + _, err := mockExecute(destDir, "git", "switch", "--detach", "origin/"+m.Revision) + if err != nil { + return fmt.Errorf("failed to switch repo to revision. repo dir: %v, revision: %v", destDir, m.Revision) + } + } + return nil } From f31bcf0887d76c72cb7fd38affb010ab5aa317fe Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Wed, 19 Apr 2023 16:12:22 -0400 Subject: [PATCH 11/15] removing git interface Signed-off-by: Michael Hoang --- go.mod | 14 ++ go.sum | 18 ++ pkg/devfile/parser/context/context.go | 16 -- pkg/devfile/parser/context/context_test.go | 3 - pkg/devfile/parser/parse.go | 41 +++- pkg/devfile/parser/parse_test.go | 207 +++++++++++++++------ pkg/git/git.go | 50 +---- pkg/git/mock.go | 56 ++---- pkg/util/util.go | 59 +++++- 9 files changed, 295 insertions(+), 169 deletions(-) diff --git a/go.mod b/go.mod index e4042050..f0d97a0f 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/devfile/registry-support/registry-library v0.0.0-20221018213054-47b3ffaeadba github.com/fatih/color v1.7.0 github.com/fsnotify/fsnotify v1.6.0 + github.com/go-git/go-git/v5 v5.4.2 github.com/gobwas/glob v0.2.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 @@ -33,6 +34,9 @@ require ( require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Microsoft/go-winio v0.5.1 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect + github.com/acomagu/bufpipe v1.0.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/containerd/containerd v1.5.9 // indirect @@ -46,7 +50,10 @@ require ( github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/emirpasic/gods v1.12.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect @@ -59,14 +66,17 @@ require ( github.com/gorilla/mux v1.8.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect github.com/klauspost/compress v1.13.6 // indirect github.com/lucasjones/reggen v0.0.0-20200904144131-37ba4fa293bb // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-colorable v0.1.2 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/reflectwalk v1.0.1 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect @@ -82,10 +92,13 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect + github.com/sergi/go-diff v1.1.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/xanzy/ssh-agent v0.3.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + golang.org/x/crypto v0.1.0 // indirect golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 // indirect golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect @@ -98,6 +111,7 @@ require ( google.golang.org/grpc v1.49.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/component-base v0.26.1 // indirect k8s.io/klog/v2 v2.80.1 // indirect diff --git a/go.sum b/go.sum index f50e9c7c..7f14589e 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,7 @@ github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:m github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -94,6 +95,7 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -101,12 +103,14 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= @@ -337,6 +341,7 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -368,11 +373,16 @@ github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXt github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= +github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -580,6 +590,7 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= @@ -602,6 +613,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -640,6 +652,7 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7 github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= @@ -662,6 +675,7 @@ github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WT github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -829,6 +843,7 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -906,6 +921,7 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -978,6 +994,7 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1444,6 +1461,7 @@ gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76 gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/devfile/parser/context/context.go b/pkg/devfile/parser/context/context.go index a9043cc6..8606a5dc 100644 --- a/pkg/devfile/parser/context/context.go +++ b/pkg/devfile/parser/context/context.go @@ -17,7 +17,6 @@ package parser import ( "fmt" - "github.com/devfile/library/v2/pkg/git" "net/url" "os" "path/filepath" @@ -52,9 +51,6 @@ type DevfileCtx struct { // token is a personal access token used with a private git repo URL token string - // Git is an interface used for git urls - git git.IGitUrl - // filesystem for devfile fs filesystem.Filesystem @@ -74,7 +70,6 @@ func NewDevfileCtx(path string) DevfileCtx { func NewURLDevfileCtx(url string) DevfileCtx { return DevfileCtx{ url: url, - git: &git.Url{}, } } @@ -83,7 +78,6 @@ func NewPrivateURLDevfileCtx(url string, token string) DevfileCtx { return DevfileCtx{ url: url, token: token, - git: &git.Url{}, } } @@ -172,11 +166,6 @@ func (d *DevfileCtx) GetToken() string { return d.token } -// GetGit func returns current git interface -func (d *DevfileCtx) GetGit() git.IGitUrl { - return d.git -} - // SetAbsPath sets absolute file path for devfile func (d *DevfileCtx) SetAbsPath() (err error) { // Set devfile absolute path @@ -198,8 +187,3 @@ func (d *DevfileCtx) GetConvertUriToInlined() bool { func (d *DevfileCtx) SetConvertUriToInlined(value bool) { d.convertUriToInlined = value } - -// SetGit sets the git interface -func (d *DevfileCtx) SetGit(git git.IGitUrl) { - d.git = git -} diff --git a/pkg/devfile/parser/context/context_test.go b/pkg/devfile/parser/context/context_test.go index 67d8c68f..32097edd 100644 --- a/pkg/devfile/parser/context/context_test.go +++ b/pkg/devfile/parser/context/context_test.go @@ -16,7 +16,6 @@ package parser import ( - "github.com/devfile/library/v2/pkg/git" "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" @@ -94,13 +93,11 @@ func TestNewURLDevfileCtx(t *testing.T) { d := NewPrivateURLDevfileCtx(url, token) assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL()) assert.Equal(t, "fake-token", d.GetToken()) - assert.Equal(t, &git.Url{}, d.GetGit()) } { d := NewURLDevfileCtx(url) assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL()) assert.Equal(t, "", d.GetToken()) - assert.Equal(t, &git.Url{}, d.GetGit()) } } diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index 69c5dfae..a88e0e38 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -47,6 +47,42 @@ import ( "github.com/pkg/errors" ) +// downloadGitRepoResources is exposed as a global variable for the purpose of running mock tests +var downloadGitRepoResources = func(url string, destDir string, httpTimeout *int, token string) error { + gitUrl, err := git.NewGitUrlWithURL(url) + if err != nil { + return err + } + + if gitUrl.IsGitProviderRepo() && gitUrl.IsFile { + stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) + if err != nil { + return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) + } + defer os.RemoveAll(stackDir) + + if !gitUrl.IsPublic(httpTimeout) { + err = gitUrl.SetToken(token, httpTimeout) + if err != nil { + return err + } + } + + err = gitUrl.CloneGitRepo(stackDir) + if err != nil { + return err + } + + dir := path.Dir(path.Join(stackDir, gitUrl.Path)) + err = git.CopyAllDirFiles(dir, destDir) + if err != nil { + return err + } + } + + return nil +} + // ParseDevfile func validates the devfile integrity. // Creates devfile context and runtime objects func parseDevfile(d DevfileObj, resolveCtx *resolutionContextTree, tool resolverTools, flattenedDevfile bool) (DevfileObj, error) { @@ -142,7 +178,6 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) { context: args.Context, k8sClient: args.K8sClient, httpTimeout: args.HTTPTimeout, - git: d.Ctx.GetGit(), } flattenedDevfile := true @@ -196,8 +231,6 @@ type resolverTools struct { k8sClient client.Client // httpTimeout is the timeout value in seconds passed in from the client. httpTimeout *int - // git is the interface used for git urls - git git.IGitUrl } func populateAndParseDevfile(d DevfileObj, resolveCtx *resolutionContextTree, tool resolverTools, flattenedDevfile bool) (DevfileObj, error) { @@ -449,7 +482,7 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D } destDir := path.Dir(curDevfileCtx.GetAbsPath()) - err = tool.git.DownloadGitRepoResources(newUri, destDir, tool.httpTimeout, token) + err = downloadGitRepoResources(newUri, destDir, tool.httpTimeout, token) if err != nil { return DevfileObj{}, err } diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index ee104a86..6412ea59 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -2859,7 +2859,8 @@ func Test_parseParentAndPluginFromURI(t *testing.T) { tt.args.devFileObj.Data.AddComponents(plugincomp) } - err := parseParentAndPlugin(tt.args.devFileObj, &resolutionContextTree{}, resolverTools{git: &git.MockGitUrl{}}) + downloadGitRepoResources = mockDownloadGitRepoResources(&git.Url{}) + err := parseParentAndPlugin(tt.args.devFileObj, &resolutionContextTree{}, resolverTools{}) // Unexpected error if (err != nil) != (tt.wantErr != nil) { @@ -3072,23 +3073,10 @@ func Test_parseParentAndPlugin_RecursivelyReference(t *testing.T) { httpTimeout := 0 - mockGitUrl := &git.MockGitUrl{ - Protocol: "https", - Host: "raw.githubusercontent.com", - Owner: "devfile", - Repo: "library", - Revision: "main", - Path: "devfile.yaml", - IsFile: true, - } - devFileObj.Ctx.SetGit(mockGitUrl) - devFileObj.Ctx.GetGit().SetToken("valid-token", &httpTimeout) - tool := resolverTools{ k8sClient: testK8sClient, context: context.Background(), httpTimeout: &httpTimeout, - git: devFileObj.Ctx.GetGit(), } err := parseParentAndPlugin(devFileObj, &resolutionContextTree{}, tool) @@ -4161,7 +4149,8 @@ func Test_parseFromURI(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseFromURI(tt.importReference, tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{git: &git.MockGitUrl{}}) + downloadGitRepoResources = mockDownloadGitRepoResources(&git.Url{}) + got, err := parseFromURI(tt.importReference, tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{}) if (err != nil) != (tt.wantErr != nil) { t.Errorf("Test_parseFromURI() unexpected error: %v, wantErr %v", err, tt.wantErr) } else if err == nil && !reflect.DeepEqual(got.Data, tt.wantDevFile.Data) { @@ -4173,15 +4162,16 @@ func Test_parseFromURI(t *testing.T) { } } -func Test_parseFromURI_GitResources(t *testing.T) { +func Test_parseFromURI_GitProviders(t *testing.T) { const ( invalidToken = "invalid-token" validToken = "valid-token" invalidRevision = "invalid-revision" ) + minimalDevfileContent := fmt.Sprintf("schemaVersion: 2.2.0") server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - _, err := rw.Write([]byte("schemaVersion: 2.2.0")) + _, err := rw.Write([]byte(minimalDevfileContent)) if err != nil { t.Error(err) } @@ -4208,9 +4198,48 @@ func Test_parseFromURI_GitResources(t *testing.T) { defer parent.Close() defer nestedParent.Close() - httpTimeoutPrivate := 1 + httpTimeout := 0 + + minimalDevfile := DevfileObj{ + Ctx: devfileCtx.NewURLDevfileCtx(OutputDevfileYamlPath), + Data: &v2.DevfileV2{ + Devfile: v1.Devfile{ + DevfileHeader: devfilepkg.DevfileHeader{ + SchemaVersion: schemaVersion, + }, + }, + }, + } + + parentDevfile := DevfileObj{ + Ctx: devfileCtx.NewURLDevfileCtx(OutputDevfileYamlPath), + Data: &v2.DevfileV2{ + Devfile: v1.Devfile{ + DevfileHeader: devfilepkg.DevfileHeader{ + SchemaVersion: schemaVersion, + Metadata: devfilepkg.DevfileMetadata{ + Name: "parent-devfile", + }, + }, + }, + }, + } + + nestParentDevfile := DevfileObj{ + Ctx: devfileCtx.NewURLDevfileCtx(OutputDevfileYamlPath), + Data: &v2.DevfileV2{ + Devfile: v1.Devfile{ + DevfileHeader: devfilepkg.DevfileHeader{ + SchemaVersion: schemaVersion, + Metadata: devfilepkg.DevfileMetadata{ + Name: "nested-devfile", + }, + }, + }, + }, + } - publicGitUrl := &git.MockGitUrl{ + publicGitUrl := &git.Url{ Protocol: "https", Host: "raw.githubusercontent.com", Owner: "devfile", @@ -4220,7 +4249,7 @@ func Test_parseFromURI_GitResources(t *testing.T) { IsFile: true, } - privateGitUrlInvalidRevision := &git.MockGitUrl{ + privateGitUrlInvalidRevision := &git.Url{ Protocol: "https", Host: "raw.githubusercontent.com", Owner: "devfile", @@ -4229,9 +4258,9 @@ func Test_parseFromURI_GitResources(t *testing.T) { Path: "devfile.yaml", IsFile: true, } - privateGitUrlInvalidRevision.SetToken(validToken, &httpTimeoutPrivate) + privateGitUrlInvalidRevision.SetToken(validToken, &httpTimeout) - privateBitbucketGitUrl := &git.MockGitUrl{ + privateBitbucketGitUrl := &git.Url{ Protocol: "https", Host: "bitbucket.org", Owner: "devfile", @@ -4240,9 +4269,9 @@ func Test_parseFromURI_GitResources(t *testing.T) { Path: "stacks/go/1.0.2/devfile.yaml", IsFile: true, } - privateBitbucketGitUrl.SetToken(validToken, &httpTimeoutPrivate) + privateBitbucketGitUrl.SetToken(validToken, &httpTimeout) - privateGitUrl := &git.MockGitUrl{ + privateGitUrl := &git.Url{ Protocol: "https", Host: "github.com", Owner: "mock-owner", @@ -4251,7 +4280,7 @@ func Test_parseFromURI_GitResources(t *testing.T) { Path: "mock/stacks/go/1.0.2/devfile.yaml", IsFile: true, } - privateGitUrl.SetToken(validToken, &httpTimeoutPrivate) + privateGitUrl.SetToken(validToken, &httpTimeout) curDevfileContextWithValidToken := devfileCtx.NewPrivateURLDevfileCtx(OutputDevfileYamlPath, validToken) curDevfileContextWithInvalidToken := devfileCtx.NewPrivateURLDevfileCtx(OutputDevfileYamlPath, invalidToken) @@ -4263,64 +4292,59 @@ func Test_parseFromURI_GitResources(t *testing.T) { tests := []struct { name string curDevfileCtx *devfileCtx.DevfileCtx - gitUrl *git.MockGitUrl - timeout *int + gitUrl *git.Url importReference v1.ImportReference + wantDevFile DevfileObj wantError *string }{ { name: "private main devfile URL", curDevfileCtx: &curDevfileContextWithValidToken, gitUrl: privateGitUrl, - timeout: &httpTimeoutPrivate, importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: server.URL, }, }, - wantError: nil, + wantDevFile: minimalDevfile, }, { name: "private main devfile Bitbucket URL", curDevfileCtx: &curDevfileContextWithValidToken, gitUrl: privateBitbucketGitUrl, - timeout: &httpTimeoutPrivate, importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: server.URL, }, }, - wantError: nil, + wantDevFile: minimalDevfile, }, { name: "private main devfile with a private parent reference", curDevfileCtx: &curDevfileContextWithValidToken, gitUrl: privateGitUrl, - timeout: &httpTimeoutPrivate, importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: parent.URL, }, }, - wantError: nil, + wantDevFile: parentDevfile, }, { name: "private main devfile with a private parent with a nested private parent reference", curDevfileCtx: &curDevfileContextWithValidToken, gitUrl: privateGitUrl, - timeout: &httpTimeoutPrivate, importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: nestedParent.URL, }, }, - wantError: nil, + wantDevFile: nestParentDevfile, }, { name: "private main devfile without a valid token", curDevfileCtx: &curDevfileContextWithInvalidToken, gitUrl: privateGitUrl, - timeout: &httpTimeoutPrivate, importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: server.URL, @@ -4332,19 +4356,17 @@ func Test_parseFromURI_GitResources(t *testing.T) { name: "public main devfile without a token", curDevfileCtx: &curDevfileContextWithoutToken, gitUrl: publicGitUrl, - timeout: &httpTimeoutPrivate, importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: server.URL, }, }, - wantError: nil, + wantDevFile: minimalDevfile, }, { name: "private parent devfile with invalid revision", curDevfileCtx: &curDevfileContextWithoutToken, gitUrl: privateGitUrlInvalidRevision, - timeout: &httpTimeoutPrivate, importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: parent.URL, @@ -4356,16 +4378,64 @@ func Test_parseFromURI_GitResources(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := parseFromURI(tt.importReference, *tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{httpTimeout: tt.timeout, git: tt.gitUrl}) + downloadGitRepoResources = mockDownloadGitRepoResources(tt.gitUrl) + got, err := parseFromURI(tt.importReference, *tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{}) if (err != nil) != (tt.wantError != nil) { t.Errorf("Unexpected error: %v, wantErr %v", err, tt.wantError) - } else if tt.wantError != nil { + } else if err == nil && !reflect.DeepEqual(got.Data, tt.wantDevFile.Data) { + t.Errorf("Wanted: %v, got: %v, difference at %v", tt.wantDevFile, got, pretty.Compare(tt.wantDevFile, got)) + } else if err != nil { assert.Regexp(t, *tt.wantError, err.Error(), "Error message should match") } }) } } +func mockDownloadGitRepoResources(gURL *git.Url) func(url string, destDir string, httpTimeout *int, token string) error { + return func(url string, destDir string, httpTimeout *int, token string) error { + // this converts the real git URL to a mock URL + mockGitUrl := git.MockGitUrl{ + Protocol: gURL.Protocol, + Host: gURL.Host, + Owner: gURL.Owner, + Repo: gURL.Repo, + Revision: gURL.Revision, + Path: gURL.Path, + IsFile: gURL.IsFile, + } + + if mockGitUrl.IsGitProviderRepo() && mockGitUrl.IsFile { + stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) + if err != nil { + return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) + } + defer os.RemoveAll(stackDir) + + err = mockGitUrl.SetToken(token) + if err != nil { + return err + } + + err = mockGitUrl.CloneGitRepo(stackDir) + if err != nil { + return err + } + + if mockGitUrl.GetToken() != "" { + _, err = os.Stat(stackDir + "/private-repo-resource.txt") + } else { + _, err = os.Stat(stackDir + "/public-repo-resource.txt") + } + + if err != nil { + return err + } + } + + return nil + } +} + func Test_parseFromRegistry(t *testing.T) { const ( registry = "127.0.0.1:8080" @@ -4701,47 +4771,66 @@ func Test_parseFromKubeCRD(t *testing.T) { } func Test_DownloadGitRepoResources(t *testing.T) { - destDir, err := ioutil.TempDir("", "") - if err != nil { - t.Errorf("Failed to create dest dir: %s, error: %v", destDir, err) + httpTimeout := 0 + + validGitUrl := git.Url{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "registry", + Revision: "main", + Path: "stacks/python/3.0.0/devfile.yaml", + IsFile: true, } - defer os.RemoveAll(destDir) + validGitUrl.SetToken("valid-token", &httpTimeout) - httpTimeout := 0 - var g git.Url + invalidTokenErr := "failed to clone repo with token, ensure that the url and token is correct" tests := []struct { name string url string + gitUrl git.Url destDir string token string wantErr bool }{ { - name: "should fail with invalid git url", - url: "non-existent-url", - destDir: path.Join(os.TempDir(), "nonexistent"), - token: "", - wantErr: true, + name: "should be able to get resources with valid token", + url: "https://raw.githubusercontent.com/devfile/registry/main/stacks/python/3.0.0/devfile.yaml", + gitUrl: validGitUrl, + token: "valid-token", + wantErr: false, }, { - name: "should be able to get resources from valid git url", + name: "should be able to get resources from public repo (empty token)", url: "https://raw.githubusercontent.com/devfile/registry/main/stacks/python/3.0.0/devfile.yaml", - destDir: destDir, + gitUrl: validGitUrl, token: "", wantErr: false, }, + { + name: "should fail to get resources with invalid token", + url: "https://raw.githubusercontent.com/devfile/registry/main/stacks/python/3.0.0/devfile.yaml", + gitUrl: validGitUrl, + token: "invalid-token", + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotErr := false - err := g.DownloadGitRepoResources(tt.url, tt.destDir, &httpTimeout, tt.token) + destDir, err := ioutil.TempDir("", "") if err != nil { - gotErr = true + t.Errorf("Failed to create dest dir: %s, error: %v", destDir, err) } - if !reflect.DeepEqual(gotErr, tt.wantErr) { - t.Errorf("Got error: %t, want error: %t", gotErr, tt.wantErr) + defer os.RemoveAll(destDir) + + downloadGitRepoResources = mockDownloadGitRepoResources(&tt.gitUrl) + err = downloadGitRepoResources(tt.url, destDir, &httpTimeout, tt.token) + if (err != nil) && (tt.wantErr != true) { + t.Errorf("Unexpected error = %v", err) + } else if tt.wantErr == true { + assert.Containsf(t, err.Error(), invalidTokenErr, "expected error containing %q, got %s", invalidTokenErr, err) } }) } diff --git a/pkg/git/git.go b/pkg/git/git.go index 5c1a751e..44a7dc47 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -17,11 +17,8 @@ package git import ( "fmt" - "io/ioutil" "net/url" - "os" "os/exec" - "path" "path/filepath" "strings" ) @@ -33,13 +30,6 @@ const ( BitbucketHost string = "bitbucket.org" ) -type IGitUrl interface { - GitRawFileAPI() string - SetToken(token string, httpTimeout *int) error - IsPublic(httpTimeout *int) bool - DownloadGitRepoResources(url string, destDir string, httpTimeout *int, token string) error -} - type Url struct { Protocol string // URL scheme Host string // URL domain name @@ -52,12 +42,12 @@ type Url struct { } // NewGitUrlWithURL NewGitUrl creates a GitUrl from a string url -func NewGitUrlWithURL(url string) (*Url, error) { +func NewGitUrlWithURL(url string) (Url, error) { gitUrl, err := ParseGitUrl(url) if err != nil { - return &gitUrl, err + return gitUrl, err } - return &gitUrl, nil + return gitUrl, nil } // ParseGitUrl extracts information from a support git url @@ -157,40 +147,6 @@ func (g *Url) CloneGitRepo(destDir string) error { return nil } -func (g *Url) DownloadGitRepoResources(url string, destDir string, httpTimeout *int, token string) error { - gitUrl, err := NewGitUrlWithURL(url) - if err != nil { - return err - } - - if gitUrl.IsGitProviderRepo() && gitUrl.IsFile { - stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) - if err != nil { - return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) - } - defer os.RemoveAll(stackDir) - - if !gitUrl.IsPublic(httpTimeout) { - err = gitUrl.SetToken(token, httpTimeout) - if err != nil { - return err - } - } - - err = gitUrl.CloneGitRepo(stackDir) - if err != nil { - return err - } - - dir := path.Dir(path.Join(stackDir, gitUrl.Path)) - err = CopyAllDirFiles(dir, destDir) - if err != nil { - return err - } - } - return nil -} - func (g *Url) parseGitHubUrl(url *url.URL) error { var splitUrl []string var err error diff --git a/pkg/git/mock.go b/pkg/git/mock.go index a6dca5b7..a6b6c77d 100644 --- a/pkg/git/mock.go +++ b/pkg/git/mock.go @@ -17,9 +17,9 @@ package git import ( "fmt" - "io/ioutil" "net/url" "os" + "path/filepath" "strings" ) @@ -44,14 +44,24 @@ var mockExecute = func(baseDir string, cmd CommandType, args ...string) ([]byte, u, _ := url.Parse(args[1]) password, hasPassword := u.User.Password() + // private repository if hasPassword { switch password { case "valid-token": + _, err := os.Create(filepath.Clean(baseDir) + "/private-repo-resource.txt") + if err != nil { + return nil, fmt.Errorf("failed to create test resource: %v", err) + } return []byte("test"), nil default: return []byte(""), fmt.Errorf("not a valid token") } } + // public repository + _, err := os.Create(filepath.Clean(baseDir) + "/public-repo-resource.txt") + if err != nil { + return nil, fmt.Errorf("failed to create test resource: %v", err) + } return []byte("test"), nil } @@ -88,7 +98,7 @@ func (m *MockGitUrl) CloneGitRepo(destDir string) error { } } - _, err := mockExecute(destDir, "git", "clone", repoUrl, ".") + _, err := mockExecute(destDir, "git", "clone", repoUrl, destDir) if err != nil { if m.GetToken() == "" { @@ -108,46 +118,16 @@ func (m *MockGitUrl) CloneGitRepo(destDir string) error { return nil } -func (m *MockGitUrl) DownloadGitRepoResources(url string, destDir string, httpTimeout *int, token string) error { - gitUrl := m - if gitUrl.IsGitProviderRepo() && gitUrl.IsFile { - stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) - if err != nil { - return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) - } - defer os.RemoveAll(stackDir) - - if !gitUrl.IsPublic(httpTimeout) { - err = m.SetToken(token, httpTimeout) - if err != nil { - return err - } - } - - err = gitUrl.CloneGitRepo(stackDir) - if err != nil { - return err - } - } - return nil -} - -func (m *MockGitUrl) SetToken(token string, httpTimeout *int) error { +func (m *MockGitUrl) SetToken(token string) error { m.token = token return nil } -func (m *MockGitUrl) IsPublic(httpTimeout *int) bool { - if *httpTimeout != 0 { +func (m *MockGitUrl) IsGitProviderRepo() bool { + switch m.Host { + case GitHubHost, RawGitHubHost, GitLabHost, BitbucketHost: + return true + default: return false } - return true -} - -func (m *MockGitUrl) GitRawFileAPI() string { - return "" -} - -func (m *MockGitUrl) IsGitProviderRepo() bool { - return true } diff --git a/pkg/util/util.go b/pkg/util/util.go index 6db76a8a..c49a542b 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -22,6 +22,8 @@ import ( "crypto/rand" "fmt" "github.com/devfile/library/v2/pkg/git" + gitpkg "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" "io" "io/ioutil" "math/big" @@ -1098,7 +1100,7 @@ func DownloadInMemory(params HTTPRequestParams) ([]byte, error) { ResponseHeaderTimeout: HTTPRequestResponseTimeout, }, Timeout: HTTPRequestResponseTimeout} - var g git.IGitUrl + var g git.Url var err error if IsGitProviderRepo(params.URL) { @@ -1111,7 +1113,7 @@ func DownloadInMemory(params HTTPRequestParams) ([]byte, error) { return downloadInMemoryWithClient(params, httpClient, g) } -func downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient, g git.IGitUrl) ([]byte, error) { +func downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient, g git.Url) ([]byte, error) { var url string url = params.URL req, err := http.NewRequest("GET", url, nil) @@ -1226,6 +1228,59 @@ func ValidateFile(filePath string) error { return nil } +// GetGitUrlComponentsFromRaw converts a raw GitHub file link to a map of the url components +// Deprecated: in favor of the method git.ParseGitUrl() with the devfile/library/v2/pkg/git package +func GetGitUrlComponentsFromRaw(rawGitURL string) (map[string]string, error) { + var urlComponents map[string]string + + err := ValidateURL(rawGitURL) + if err != nil { + return nil, err + } + + u, _ := url.Parse(rawGitURL) + // the url scheme (e.g. https://) is removed before splitting into the 5 components + urlPath := strings.SplitN(u.Host+u.Path, "/", 5) + + // raw GitHub url: https://raw.githubusercontent.com/devfile/registry/main/stacks/nodejs/devfile.yaml + // host: raw.githubusercontent.com + // username: devfile + // project: registry + // branch: main + // file: stacks/nodejs/devfile.yaml + if len(urlPath) == 5 { + urlComponents = map[string]string{ + "host": urlPath[0], + "username": urlPath[1], + "project": urlPath[2], + "branch": urlPath[3], + "file": urlPath[4], + } + } + + return urlComponents, nil +} + +// CloneGitRepo clones a GitHub repo to a destination directory +// Deprecated: in favor of the method git.CloneGitRepo() with the devfile/library/v2/pkg/git package +func CloneGitRepo(gitUrlComponents map[string]string, destDir string) error { + gitUrl := fmt.Sprintf("https://github.com/%s/%s.git", gitUrlComponents["username"], gitUrlComponents["project"]) + branch := fmt.Sprintf("refs/heads/%s", gitUrlComponents["branch"]) + + cloneOptions := &gitpkg.CloneOptions{ + URL: gitUrl, + ReferenceName: plumbing.ReferenceName(branch), + SingleBranch: true, + Depth: 1, + } + + _, err := gitpkg.PlainClone(destDir, false, cloneOptions) + if err != nil { + return err + } + return nil +} + // CopyFile copies file from source path to destination path func CopyFile(srcPath string, dstPath string, info os.FileInfo) error { // In order to avoid file overriding issue, do nothing if source path is equal to destination path From ca38edf8971569de085fc9e2acc358a4ea0f1c70 Mon Sep 17 00:00:00 2001 From: Kim Tsao <84398375+kim-tsao@users.noreply.github.com> Date: Fri, 21 Apr 2023 15:13:50 -0400 Subject: [PATCH 12/15] update license header to current year --- pkg/devfile/parser/context/content.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/devfile/parser/context/content.go b/pkg/devfile/parser/context/content.go index 1347b75a..52b3a45c 100644 --- a/pkg/devfile/parser/context/content.go +++ b/pkg/devfile/parser/context/content.go @@ -1,5 +1,5 @@ // -// Copyright 2022 Red Hat, Inc. +// Copyright 2022-2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From 7fbbc0b272d6411a5d2859a58cdf81a5cd3f2b67 Mon Sep 17 00:00:00 2001 From: Kim Tsao <84398375+kim-tsao@users.noreply.github.com> Date: Fri, 21 Apr 2023 15:15:01 -0400 Subject: [PATCH 13/15] update license header to current year --- pkg/devfile/parser/context/context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/devfile/parser/context/context.go b/pkg/devfile/parser/context/context.go index 8606a5dc..d0589315 100644 --- a/pkg/devfile/parser/context/context.go +++ b/pkg/devfile/parser/context/context.go @@ -1,5 +1,5 @@ // -// Copyright 2022 Red Hat, Inc. +// Copyright 2022-2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From b2357c7967d46a727f81f9bb58a31283648e6d2c Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Fri, 21 Apr 2023 15:27:23 -0400 Subject: [PATCH 14/15] updating licence header year Signed-off-by: Michael Hoang --- pkg/devfile/parser/context/context_test.go | 2 +- pkg/git/util_test.go | 2 +- pkg/util/util.go | 2 +- pkg/util/util_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/devfile/parser/context/context_test.go b/pkg/devfile/parser/context/context_test.go index 32097edd..f1dc8292 100644 --- a/pkg/devfile/parser/context/context_test.go +++ b/pkg/devfile/parser/context/context_test.go @@ -1,5 +1,5 @@ // -// Copyright 2022 Red Hat, Inc. +// Copyright 2022-2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/git/util_test.go b/pkg/git/util_test.go index 3fd3126e..3c0f54c3 100644 --- a/pkg/git/util_test.go +++ b/pkg/git/util_test.go @@ -1,5 +1,5 @@ // -// Copyright 2021-2022 Red Hat, Inc. +// Copyright 2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/util/util.go b/pkg/util/util.go index c49a542b..aa490a30 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1,5 +1,5 @@ // -// Copyright 2022 Red Hat, Inc. +// Copyright 2022-2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index fb01e4fc..55a8bdae 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -1,5 +1,5 @@ // -// Copyright 2021-2022 Red Hat, Inc. +// Copyright 2021-2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From 7acc246243b02dd22acc1b3a3f63cff0371e0d59 Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Thu, 27 Apr 2023 16:46:15 -0400 Subject: [PATCH 15/15] updating tests Signed-off-by: Michael Hoang --- README.md | 23 +- pkg/devfile/parser/context/context.go | 13 +- pkg/devfile/parser/context/context_test.go | 8 +- pkg/devfile/parser/parse.go | 43 ++- pkg/devfile/parser/parse_test.go | 357 +++++++++++---------- pkg/git/git.go | 37 ++- pkg/git/git_test.go | 140 ++++---- pkg/git/mock.go | 27 +- pkg/util/util.go | 5 +- 9 files changed, 344 insertions(+), 309 deletions(-) diff --git a/README.md b/README.md index f5352480..17efc7de 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,9 @@ The Devfile Parser library is a Golang module that: 2. writes to the devfile.yaml with the updated data. 3. generates Kubernetes objects for the various devfile resources. 4. defines util functions for the devfile. +5. downloads resources from a parent devfile if specified in the devfile.yaml -## Private Repository Support +## Private repository support Tokens are required to be set in the following cases: 1. parsing a devfile from a private repository @@ -24,19 +25,28 @@ Set the token for the repository: ```go parser.ParserArgs{ ... - URL: + // URL must point to a devfile.yaml + URL: /devfile.yaml Token: ... } ``` Note: The url must also be set with a supported git provider repo url. +Minimum token scope required: +1. GitHub: Read access to code +2. GitLab: Read repository +3. Bitbucket: Read repository + +Note: To select token scopes for GitHub, a fine-grained token is required. + For more information about personal access tokens: 1. [GitHub docs](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) 2. [GitLab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token) 3. [Bitbucket docs](https://support.atlassian.com/bitbucket-cloud/docs/repository-access-tokens/) [1] Currently, this works under the assumption that the token can authenticate the devfile and the parent devfile; both devfiles are in the same repository. + [2] In this scenario, the token will be used to authenticate the main devfile. ## Usage @@ -199,6 +209,15 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g } ``` +9. When parsing a devfile that contains a parent reference, if the parent uri is a supported git provider repo url with the correct personal access token, all resources from the parent git repo excluding the parent devfile.yaml will be downloaded to the location of the devfile being parsed. **Note: The URL must point to a devfile.yaml** + ```yaml + schemaVersion: 2.2.0 + ... + parent: + uri: /devfile.yaml + ... + ``` + ## Projects using devfile/library The following projects are consuming this library as a Golang dependency diff --git a/pkg/devfile/parser/context/context.go b/pkg/devfile/parser/context/context.go index d0589315..97ab49d3 100644 --- a/pkg/devfile/parser/context/context.go +++ b/pkg/devfile/parser/context/context.go @@ -73,14 +73,6 @@ func NewURLDevfileCtx(url string) DevfileCtx { } } -// NewPrivateURLDevfileCtx returns a new DevfileCtx type object -func NewPrivateURLDevfileCtx(url string, token string) DevfileCtx { - return DevfileCtx{ - url: url, - token: token, - } -} - // NewByteContentDevfileCtx set devfile content from byte data and returns a new DevfileCtx type object and error func NewByteContentDevfileCtx(data []byte) (d DevfileCtx, err error) { err = d.SetDevfileContentFromBytes(data) @@ -166,6 +158,11 @@ func (d *DevfileCtx) GetToken() string { return d.token } +// SetToken sets the token for the devfile +func (d *DevfileCtx) SetToken(token string) { + d.token = token +} + // SetAbsPath sets absolute file path for devfile func (d *DevfileCtx) SetAbsPath() (err error) { // Set devfile absolute path diff --git a/pkg/devfile/parser/context/context_test.go b/pkg/devfile/parser/context/context_test.go index f1dc8292..3f332e88 100644 --- a/pkg/devfile/parser/context/context_test.go +++ b/pkg/devfile/parser/context/context_test.go @@ -88,16 +88,12 @@ func TestNewURLDevfileCtx(t *testing.T) { token = "fake-token" url = "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml" ) - - { - d := NewPrivateURLDevfileCtx(url, token) - assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL()) - assert.Equal(t, "fake-token", d.GetToken()) - } { d := NewURLDevfileCtx(url) assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL()) assert.Equal(t, "", d.GetToken()) + d.SetToken(token) + assert.Equal(t, "fake-token", d.GetToken()) } } diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index a88e0e38..e22fa0eb 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" "github.com/devfile/library/v2/pkg/git" + "github.com/hashicorp/go-multierror" "io/ioutil" "net/url" "os" @@ -49,34 +50,49 @@ import ( // downloadGitRepoResources is exposed as a global variable for the purpose of running mock tests var downloadGitRepoResources = func(url string, destDir string, httpTimeout *int, token string) error { + var returnedErr error + gitUrl, err := git.NewGitUrlWithURL(url) if err != nil { return err } - if gitUrl.IsGitProviderRepo() && gitUrl.IsFile { - stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) + if gitUrl.IsGitProviderRepo() { + if !gitUrl.IsFile || gitUrl.Revision == "" || !strings.Contains(gitUrl.Path, OutputDevfileYamlPath) { + return fmt.Errorf("error getting devfile from url: failed to retrieve %s", url) + } + + stackDir, err := os.MkdirTemp("", fmt.Sprintf("git-resources")) if err != nil { return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) } - defer os.RemoveAll(stackDir) + + defer func(path string) { + err := os.RemoveAll(path) + if err != nil { + returnedErr = multierror.Append(returnedErr, err) + } + }(stackDir) if !gitUrl.IsPublic(httpTimeout) { err = gitUrl.SetToken(token, httpTimeout) if err != nil { - return err + returnedErr = multierror.Append(returnedErr, err) + return returnedErr } } err = gitUrl.CloneGitRepo(stackDir) if err != nil { - return err + returnedErr = multierror.Append(returnedErr, err) + return returnedErr } dir := path.Dir(path.Join(stackDir, gitUrl.Path)) err = git.CopyAllDirFiles(dir, destDir) if err != nil { - return err + returnedErr = multierror.Append(returnedErr, err) + return returnedErr } } @@ -163,15 +179,15 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) { } else if args.Path != "" { d.Ctx = devfileCtx.NewDevfileCtx(args.Path) } else if args.URL != "" { - if args.Token != "" { - d.Ctx = devfileCtx.NewPrivateURLDevfileCtx(args.URL, args.Token) - } else { - d.Ctx = devfileCtx.NewURLDevfileCtx(args.URL) - } + d.Ctx = devfileCtx.NewURLDevfileCtx(args.URL) } else { return d, errors.Wrap(err, "the devfile source is not provided") } + if args.Token != "" { + d.Ctx.SetToken(args.Token) + } + tool := resolverTools{ defaultNamespace: args.DefaultNamespace, registryURLs: args.RegistryURLs, @@ -475,10 +491,9 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D } token := curDevfileCtx.GetToken() + d.Ctx = devfileCtx.NewURLDevfileCtx(newUri) if token != "" { - d.Ctx = devfileCtx.NewPrivateURLDevfileCtx(newUri, token) - } else { - d.Ctx = devfileCtx.NewURLDevfileCtx(newUri) + d.Ctx.SetToken(token) } destDir := path.Dir(curDevfileCtx.GetAbsPath()) diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index 6412ea59..0971f8d9 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -16,6 +16,7 @@ package parser import ( + "bytes" "context" "fmt" v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" @@ -26,6 +27,7 @@ import ( "net/http/httptest" "os" "path" + "path/filepath" "reflect" "strings" "testing" @@ -2859,7 +2861,7 @@ func Test_parseParentAndPluginFromURI(t *testing.T) { tt.args.devFileObj.Data.AddComponents(plugincomp) } - downloadGitRepoResources = mockDownloadGitRepoResources(&git.Url{}) + downloadGitRepoResources = mockDownloadGitRepoResources(&git.GitUrl{}, "") err := parseParentAndPlugin(tt.args.devFileObj, &resolutionContextTree{}, resolverTools{}) // Unexpected error @@ -3079,6 +3081,7 @@ func Test_parseParentAndPlugin_RecursivelyReference(t *testing.T) { httpTimeout: &httpTimeout, } + downloadGitRepoResources = mockDownloadGitRepoResources(&git.GitUrl{}, "") err := parseParentAndPlugin(devFileObj, &resolutionContextTree{}, tool) // devfile has a cycle in references: main devfile -> uri: http://127.0.0.1:8080 -> name: testcrd, namespace: defaultnamespace -> uri: http://127.0.0.1:8090 -> uri: http://127.0.0.1:8080 expectedErr := fmt.Sprintf("devfile has an cycle in references: main devfile -> uri: %s%s -> name: %s, namespace: %s -> uri: %s%s -> uri: %s%s", httpPrefix, uri1, name, namespace, @@ -4149,7 +4152,7 @@ func Test_parseFromURI(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - downloadGitRepoResources = mockDownloadGitRepoResources(&git.Url{}) + downloadGitRepoResources = mockDownloadGitRepoResources(&git.GitUrl{}, "") got, err := parseFromURI(tt.importReference, tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{}) if (err != nil) != (tt.wantErr != nil) { t.Errorf("Test_parseFromURI() unexpected error: %v, wantErr %v", err, tt.wantErr) @@ -4169,77 +4172,30 @@ func Test_parseFromURI_GitProviders(t *testing.T) { invalidRevision = "invalid-revision" ) - minimalDevfileContent := fmt.Sprintf("schemaVersion: 2.2.0") + minimalDevfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: devfile") server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { _, err := rw.Write([]byte(minimalDevfileContent)) if err != nil { t.Error(err) } })) - - parentDevfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: parent-devfile\nparent:\n uri: \"%s\"", server.URL) - parent := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - _, err := rw.Write([]byte(parentDevfileContent)) - if err != nil { - t.Error(err) - } - })) - - devfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: nested-devfile\nparent:\n uri: \"%s\"", parent.URL) - nestedParent := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - _, err := rw.Write([]byte(devfileContent)) - if err != nil { - t.Error(err) - } - })) - - // Close the server when test finishes defer server.Close() - defer parent.Close() - defer nestedParent.Close() - - httpTimeout := 0 minimalDevfile := DevfileObj{ - Ctx: devfileCtx.NewURLDevfileCtx(OutputDevfileYamlPath), - Data: &v2.DevfileV2{ - Devfile: v1.Devfile{ - DevfileHeader: devfilepkg.DevfileHeader{ - SchemaVersion: schemaVersion, - }, - }, - }, - } - - parentDevfile := DevfileObj{ Ctx: devfileCtx.NewURLDevfileCtx(OutputDevfileYamlPath), Data: &v2.DevfileV2{ Devfile: v1.Devfile{ DevfileHeader: devfilepkg.DevfileHeader{ SchemaVersion: schemaVersion, Metadata: devfilepkg.DevfileMetadata{ - Name: "parent-devfile", + Name: "devfile", }, }, }, }, } - nestParentDevfile := DevfileObj{ - Ctx: devfileCtx.NewURLDevfileCtx(OutputDevfileYamlPath), - Data: &v2.DevfileV2{ - Devfile: v1.Devfile{ - DevfileHeader: devfilepkg.DevfileHeader{ - SchemaVersion: schemaVersion, - Metadata: devfilepkg.DevfileMetadata{ - Name: "nested-devfile", - }, - }, - }, - }, - } - - publicGitUrl := &git.Url{ + validGitUrl := &git.GitUrl{ Protocol: "https", Host: "raw.githubusercontent.com", Owner: "devfile", @@ -4249,137 +4205,156 @@ func Test_parseFromURI_GitProviders(t *testing.T) { IsFile: true, } - privateGitUrlInvalidRevision := &git.Url{ - Protocol: "https", - Host: "raw.githubusercontent.com", - Owner: "devfile", - Repo: "library", - Revision: invalidRevision, - Path: "devfile.yaml", - IsFile: true, - } - privateGitUrlInvalidRevision.SetToken(validToken, &httpTimeout) - - privateBitbucketGitUrl := &git.Url{ - Protocol: "https", - Host: "bitbucket.org", - Owner: "devfile", - Repo: "registry", - Revision: "main", - Path: "stacks/go/1.0.2/devfile.yaml", - IsFile: true, - } - privateBitbucketGitUrl.SetToken(validToken, &httpTimeout) - - privateGitUrl := &git.Url{ - Protocol: "https", - Host: "github.com", - Owner: "mock-owner", - Repo: "mock-repo", - Revision: "mock-branch", - Path: "mock/stacks/go/1.0.2/devfile.yaml", - IsFile: true, - } - privateGitUrl.SetToken(validToken, &httpTimeout) - - curDevfileContextWithValidToken := devfileCtx.NewPrivateURLDevfileCtx(OutputDevfileYamlPath, validToken) - curDevfileContextWithInvalidToken := devfileCtx.NewPrivateURLDevfileCtx(OutputDevfileYamlPath, invalidToken) - curDevfileContextWithoutToken := devfileCtx.NewURLDevfileCtx(OutputDevfileYamlPath) - invalidTokenError := "failed to clone repo with token, ensure that the url and token is correct" invalidGitSwitchError := "failed to switch repo to revision*" + invalidDevfilePathError := "error getting devfile from url: failed to retrieve*" tests := []struct { - name string - curDevfileCtx *devfileCtx.DevfileCtx - gitUrl *git.Url - importReference v1.ImportReference - wantDevFile DevfileObj - wantError *string + name string + gitUrl *git.GitUrl + token string + destDir string + importReference v1.ImportReference + wantDevFile DevfileObj + wantError *string + wantResources []string + wantResourceContent []byte }{ { - name: "private main devfile URL", - curDevfileCtx: &curDevfileContextWithValidToken, - gitUrl: privateGitUrl, + name: "private parent devfile", + gitUrl: validGitUrl, + token: validToken, importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: server.URL, }, }, - wantDevFile: minimalDevfile, + wantDevFile: minimalDevfile, + wantResources: []string{"resource.file"}, + wantResourceContent: []byte("private repo\ngit switched"), }, { - name: "private main devfile Bitbucket URL", - curDevfileCtx: &curDevfileContextWithValidToken, - gitUrl: privateBitbucketGitUrl, + name: "public parent devfile", + gitUrl: validGitUrl, + token: "", importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: server.URL, }, }, - wantDevFile: minimalDevfile, + wantDevFile: minimalDevfile, + wantResources: []string{"resource.file"}, + wantResourceContent: []byte("public repo\ngit switched"), }, { - name: "private main devfile with a private parent reference", - curDevfileCtx: &curDevfileContextWithValidToken, - gitUrl: privateGitUrl, + // a valid parent url must contain a revision + name: "private parent devfile without a revision", + gitUrl: &git.GitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "library", + Revision: "", + Path: "devfile.yaml", + IsFile: true, + }, + token: validToken, importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ - Uri: parent.URL, + Uri: server.URL, }, }, - wantDevFile: parentDevfile, + wantError: &invalidDevfilePathError, + wantResources: []string{}, }, { - name: "private main devfile with a private parent with a nested private parent reference", - curDevfileCtx: &curDevfileContextWithValidToken, - gitUrl: privateGitUrl, + name: "public parent devfile with no devfile path", + gitUrl: &git.GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + IsFile: false, + }, + token: "", importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ - Uri: nestedParent.URL, + Uri: server.URL, }, }, - wantDevFile: nestParentDevfile, + wantError: &invalidDevfilePathError, + wantResources: []string{}, }, { - name: "private main devfile without a valid token", - curDevfileCtx: &curDevfileContextWithInvalidToken, - gitUrl: privateGitUrl, + name: "public parent devfile with invalid devfile path", + gitUrl: &git.GitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "library", + Revision: "main", + Path: "text.txt", + IsFile: true, + }, + token: "", importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: server.URL, }, }, - wantError: &invalidTokenError, + wantError: &invalidDevfilePathError, + wantResources: []string{}, }, { - name: "public main devfile without a token", - curDevfileCtx: &curDevfileContextWithoutToken, - gitUrl: publicGitUrl, + name: "private parent devfile with invalid token", + gitUrl: validGitUrl, + token: invalidToken, importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: server.URL, }, }, - wantDevFile: minimalDevfile, + wantError: &invalidTokenError, + wantResources: []string{}, }, { - name: "private parent devfile with invalid revision", - curDevfileCtx: &curDevfileContextWithoutToken, - gitUrl: privateGitUrlInvalidRevision, + name: "private parent devfile with invalid revision", + gitUrl: &git.GitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "library", + Revision: invalidRevision, + Path: "devfile.yaml", + IsFile: true, + }, + token: validToken, importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ - Uri: parent.URL, + Uri: server.URL, }, }, - wantError: &invalidGitSwitchError, + wantError: &invalidGitSwitchError, + wantResources: []string{}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - downloadGitRepoResources = mockDownloadGitRepoResources(tt.gitUrl) - got, err := parseFromURI(tt.importReference, *tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{}) + destDir := t.TempDir() + curDevfileContext := devfileCtx.NewDevfileCtx(path.Join(destDir, OutputDevfileYamlPath)) + err := curDevfileContext.SetAbsPath() + if err != nil { + t.Errorf("Unexpected err: %+v", err) + } + + // tt.gitUrl is the parent devfile URL + downloadGitRepoResources = mockDownloadGitRepoResources(tt.gitUrl, tt.token) + got, err := parseFromURI(tt.importReference, curDevfileContext, &resolutionContextTree{}, resolverTools{}) + + // validate even if we want an error; check that no files are copied to destDir + validateGitResourceFunctions(t, tt.wantResources, tt.wantResourceContent, destDir) + if (err != nil) != (tt.wantError != nil) { t.Errorf("Unexpected error: %v, wantErr %v", err, tt.wantError) } else if err == nil && !reflect.DeepEqual(got.Data, tt.wantDevFile.Data) { @@ -4391,7 +4366,41 @@ func Test_parseFromURI_GitProviders(t *testing.T) { } } -func mockDownloadGitRepoResources(gURL *git.Url) func(url string, destDir string, httpTimeout *int, token string) error { +// copied from: https://github.com/devfile/registry-support/blob/main/registry-library/library/library_test.go#L1118 +func validateGitResourceFunctions(t *testing.T, wantFiles []string, wantResourceContent []byte, path string) { + wantNumFiles := len(wantFiles) + files, err := os.ReadDir(path) + if err != nil { + if wantNumFiles != 0 { + t.Errorf("error reading directory %s", path) + } + } else { + // verify only the expected number of files are downloaded + gotNumFiles := len(files) + if gotNumFiles != wantNumFiles { + t.Errorf("The number of downloaded files do not match, want %d got %d", wantNumFiles, gotNumFiles) + } + // verify the expected resources are copied to the dest directory + for _, wantFile := range wantFiles { + if _, err = os.Stat(path + "/" + wantFile); err != nil && os.IsNotExist(err) { + t.Errorf("file %s should exist ", wantFile) + } + } + + // verify contents of resource file; don't need to check if wantResourceContent is nil + if wantResourceContent != nil { + resourceContent, err := os.ReadFile(filepath.Clean(path) + "/resource.file") + if err != nil { + t.Errorf("failed to open test resource: %v", err) + } + if !bytes.Equal(resourceContent, wantResourceContent) { + t.Errorf("Wanted resource content:\n%v\ngot:\n%v\ndifference at\n%v", wantResourceContent, resourceContent, pretty.Compare(string(wantResourceContent), string(resourceContent))) + } + } + } +} + +func mockDownloadGitRepoResources(gURL *git.GitUrl, mockToken string) func(url string, destDir string, httpTimeout *int, token string) error { return func(url string, destDir string, httpTimeout *int, token string) error { // this converts the real git URL to a mock URL mockGitUrl := git.MockGitUrl{ @@ -4404,14 +4413,24 @@ func mockDownloadGitRepoResources(gURL *git.Url) func(url string, destDir string IsFile: gURL.IsFile, } - if mockGitUrl.IsGitProviderRepo() && mockGitUrl.IsFile { - stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) + if mockGitUrl.IsGitProviderRepo() { + if !mockGitUrl.IsFile || mockGitUrl.Revision == "" || !strings.Contains(mockGitUrl.Path, OutputDevfileYamlPath) { + return fmt.Errorf("error getting devfile from url: failed to retrieve %s", url+"/"+mockGitUrl.Path) + } + + stackDir, err := os.MkdirTemp("", fmt.Sprintf("git-resources")) if err != nil { return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) } - defer os.RemoveAll(stackDir) - err = mockGitUrl.SetToken(token) + defer func(path string) { + err := os.RemoveAll(path) + if err != nil { + err = fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) + } + }(stackDir) + + err = mockGitUrl.SetToken(mockToken) if err != nil { return err } @@ -4421,12 +4440,7 @@ func mockDownloadGitRepoResources(gURL *git.Url) func(url string, destDir string return err } - if mockGitUrl.GetToken() != "" { - _, err = os.Stat(stackDir + "/private-repo-resource.txt") - } else { - _, err = os.Stat(stackDir + "/public-repo-resource.txt") - } - + err = git.CopyAllDirFiles(stackDir, destDir) if err != nil { return err } @@ -4773,7 +4787,7 @@ func Test_parseFromKubeCRD(t *testing.T) { func Test_DownloadGitRepoResources(t *testing.T) { httpTimeout := 0 - validGitUrl := git.Url{ + validGitUrl := git.GitUrl{ Protocol: "https", Host: "raw.githubusercontent.com", Owner: "devfile", @@ -4782,55 +4796,58 @@ func Test_DownloadGitRepoResources(t *testing.T) { Path: "stacks/python/3.0.0/devfile.yaml", IsFile: true, } - validGitUrl.SetToken("valid-token", &httpTimeout) invalidTokenErr := "failed to clone repo with token, ensure that the url and token is correct" tests := []struct { - name string - url string - gitUrl git.Url - destDir string - token string - wantErr bool + name string + url string + gitUrl git.GitUrl + destDir string + token string + wantErr bool + wantResources []string + wantResourceContent []byte }{ { - name: "should be able to get resources with valid token", - url: "https://raw.githubusercontent.com/devfile/registry/main/stacks/python/3.0.0/devfile.yaml", - gitUrl: validGitUrl, - token: "valid-token", - wantErr: false, + name: "should be able to get resources with valid token", + url: "https://raw.githubusercontent.com/devfile/registry/main/stacks/python/3.0.0/devfile.yaml", + gitUrl: validGitUrl, + token: "valid-token", + wantErr: false, + wantResources: []string{"resource.file"}, + wantResourceContent: []byte("private repo\ngit switched"), }, { - name: "should be able to get resources from public repo (empty token)", - url: "https://raw.githubusercontent.com/devfile/registry/main/stacks/python/3.0.0/devfile.yaml", - gitUrl: validGitUrl, - token: "", - wantErr: false, + name: "should be able to get resources from public repo (empty token)", + url: "https://raw.githubusercontent.com/devfile/registry/main/stacks/python/3.0.0/devfile.yaml", + gitUrl: validGitUrl, + token: "", + wantErr: false, + wantResources: []string{"resource.file"}, + wantResourceContent: []byte("public repo\ngit switched"), }, { - name: "should fail to get resources with invalid token", - url: "https://raw.githubusercontent.com/devfile/registry/main/stacks/python/3.0.0/devfile.yaml", - gitUrl: validGitUrl, - token: "invalid-token", - wantErr: true, + name: "should fail to get resources with invalid token", + url: "https://raw.githubusercontent.com/devfile/registry/main/stacks/python/3.0.0/devfile.yaml", + gitUrl: validGitUrl, + token: "invalid-token", + wantErr: true, + wantResources: []string{}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - destDir, err := ioutil.TempDir("", "") - if err != nil { - t.Errorf("Failed to create dest dir: %s, error: %v", destDir, err) - } - defer os.RemoveAll(destDir) - - downloadGitRepoResources = mockDownloadGitRepoResources(&tt.gitUrl) - err = downloadGitRepoResources(tt.url, destDir, &httpTimeout, tt.token) + destDir := t.TempDir() + downloadGitRepoResources = mockDownloadGitRepoResources(&tt.gitUrl, tt.token) + err := downloadGitRepoResources(tt.url, destDir, &httpTimeout, tt.token) if (err != nil) && (tt.wantErr != true) { t.Errorf("Unexpected error = %v", err) } else if tt.wantErr == true { assert.Containsf(t, err.Error(), invalidTokenErr, "expected error containing %q, got %s", invalidTokenErr, err) + } else { + validateGitResourceFunctions(t, tt.wantResources, tt.wantResourceContent, destDir) } }) } diff --git a/pkg/git/git.go b/pkg/git/git.go index 44a7dc47..94c7680e 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -18,6 +18,7 @@ package git import ( "fmt" "net/url" + "os" "os/exec" "path/filepath" "strings" @@ -30,19 +31,19 @@ const ( BitbucketHost string = "bitbucket.org" ) -type Url struct { +type GitUrl struct { Protocol string // URL scheme Host string // URL domain name Owner string // name of the repo owner Repo string // name of the repo Revision string // branch name, tag name, or commit id Path string // path to a directory or file in the repo - token string // used for authenticating a private repo + token string // authenticates private repo actions for parent devfiles IsFile bool // defines if the URL points to a file in the repo } // NewGitUrlWithURL NewGitUrl creates a GitUrl from a string url -func NewGitUrlWithURL(url string) (Url, error) { +func NewGitUrlWithURL(url string) (GitUrl, error) { gitUrl, err := ParseGitUrl(url) if err != nil { return gitUrl, err @@ -52,8 +53,8 @@ func NewGitUrlWithURL(url string) (Url, error) { // ParseGitUrl extracts information from a support git url // Only supports git repositories hosted on GitHub, GitLab, and Bitbucket -func ParseGitUrl(fullUrl string) (Url, error) { - var g Url +func ParseGitUrl(fullUrl string) (GitUrl, error) { + var g GitUrl err := ValidateURL(fullUrl) if err != nil { return g, err @@ -81,7 +82,7 @@ func ParseGitUrl(fullUrl string) (Url, error) { return g, err } -func (g *Url) GetToken() string { +func (g *GitUrl) GetToken() string { return g.token } @@ -106,7 +107,7 @@ var execute = func(baseDir string, cmd CommandType, args ...string) ([]byte, err return []byte(""), fmt.Errorf(unsupportedCmdMsg, string(cmd)) } -func (g *Url) CloneGitRepo(destDir string) error { +func (g *GitUrl) CloneGitRepo(destDir string) error { exist := CheckPathExists(destDir) if !exist { return fmt.Errorf("failed to clone repo, destination directory: '%s' does not exists", destDir) @@ -140,6 +141,10 @@ func (g *Url) CloneGitRepo(destDir string) error { if g.Revision != "" { _, err := execute(destDir, "git", "switch", "--detach", "origin/"+g.Revision) if err != nil { + err = os.RemoveAll(destDir) + if err != nil { + return err + } return fmt.Errorf("failed to switch repo to revision. repo dir: %v, revision: %v", destDir, g.Revision) } } @@ -147,7 +152,7 @@ func (g *Url) CloneGitRepo(destDir string) error { return nil } -func (g *Url) parseGitHubUrl(url *url.URL) error { +func (g *GitUrl) parseGitHubUrl(url *url.URL) error { var splitUrl []string var err error @@ -209,7 +214,7 @@ func (g *Url) parseGitHubUrl(url *url.URL) error { return err } -func (g *Url) parseGitLabUrl(url *url.URL) error { +func (g *GitUrl) parseGitLabUrl(url *url.URL) error { var splitFile, splitOrg []string var err error @@ -257,7 +262,7 @@ func (g *Url) parseGitLabUrl(url *url.URL) error { return err } -func (g *Url) parseBitbucketUrl(url *url.URL) error { +func (g *GitUrl) parseBitbucketUrl(url *url.URL) error { var splitUrl []string var err error @@ -295,7 +300,7 @@ func (g *Url) parseBitbucketUrl(url *url.URL) error { // SetToken validates the token with a get request to the repo before setting the token // Defaults token to empty on failure. -func (g *Url) SetToken(token string, httpTimeout *int) error { +func (g *GitUrl) SetToken(token string, httpTimeout *int) error { err := g.validateToken(HTTPRequestParams{Token: token, Timeout: httpTimeout}) if err != nil { g.token = "" @@ -307,7 +312,7 @@ func (g *Url) SetToken(token string, httpTimeout *int) error { // IsPublic checks if the GitUrl is public with a get request to the repo using an empty token // Returns true if the request succeeds -func (g *Url) IsPublic(httpTimeout *int) bool { +func (g *GitUrl) IsPublic(httpTimeout *int) bool { err := g.validateToken(HTTPRequestParams{Token: "", Timeout: httpTimeout}) if err != nil { return false @@ -317,7 +322,7 @@ func (g *Url) IsPublic(httpTimeout *int) bool { // validateToken makes a http get request to the repo with the GitUrl token // Returns an error if the get request fails -func (g *Url) validateToken(params HTTPRequestParams) error { +func (g *GitUrl) validateToken(params HTTPRequestParams) error { var apiUrl string switch g.Host { @@ -341,14 +346,14 @@ func (g *Url) validateToken(params HTTPRequestParams) error { } // GitRawFileAPI returns the endpoint for the git providers raw file -func (g *Url) GitRawFileAPI() string { +func (g *GitUrl) GitRawFileAPI() string { var apiRawFile string switch g.Host { case GitHubHost, RawGitHubHost: apiRawFile = fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", g.Owner, g.Repo, g.Revision, g.Path) case GitLabHost: - apiRawFile = fmt.Sprintf("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/files/%s/raw", g.Owner, g.Repo, g.Path) + apiRawFile = fmt.Sprintf("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/files/%s/raw?ref=%s", g.Owner, g.Repo, g.Path, g.Revision) case BitbucketHost: apiRawFile = fmt.Sprintf("https://api.bitbucket.org/2.0/repositories/%s/%s/src/%s/%s", g.Owner, g.Repo, g.Revision, g.Path) } @@ -357,7 +362,7 @@ func (g *Url) GitRawFileAPI() string { } // IsGitProviderRepo checks if the url matches a repo from a supported git provider -func (g *Url) IsGitProviderRepo() bool { +func (g *GitUrl) IsGitProviderRepo() bool { switch g.Host { case GitHubHost, RawGitHubHost, GitLabHost, BitbucketHost: return true diff --git a/pkg/git/git_test.go b/pkg/git/git_test.go index a958185d..09116fef 100644 --- a/pkg/git/git_test.go +++ b/pkg/git/git_test.go @@ -25,27 +25,41 @@ import ( ) func Test_ParseGitUrl(t *testing.T) { + invalidUrlError := "URL is invalid" + invalidUrlPathError := "url path to directory or file should contain*" + missingUserAndRepoError := "url path should contain /*" + + invalidGitHostError := "url host should be a valid GitHub, GitLab, or Bitbucket host*" + invalidGitHubPathError := "url path should contain ////*" + invalidGitHubRawPathError := "raw url path should contain ///*" + + invalidGitLabPathError := "url path to directory or file should contain //*" + missingGitLabKeywordError := "url path should contain 'blob' or 'tree' or 'raw'*" + + invalidBitbucketPathError := "url path should contain path to directory or file*" + missingBitbucketKeywordError := "url path should contain 'raw' or 'src'*" + tests := []struct { name string url string - wantUrl Url + wantUrl GitUrl wantErr string }{ { name: "should fail with empty url", url: "", - wantErr: "URL is invalid", + wantErr: invalidUrlError, }, { name: "should fail with invalid git host", url: "https://google.ca/", - wantErr: "url host should be a valid GitHub, GitLab, or Bitbucket host*", + wantErr: invalidGitHostError, }, // GitHub { name: "should parse GitHub repo with root path", url: "https://github.com/devfile/library", - wantUrl: Url{ + wantUrl: GitUrl{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -58,7 +72,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse GitHub repo with root path and tag", url: "https://github.com/devfile/library/tree/v2.2.0", - wantUrl: Url{ + wantUrl: GitUrl{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -71,7 +85,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse GitHub repo with root path and revision", url: "https://github.com/devfile/library/tree/0ce592a416fb185564516353891a45016ac7f671", - wantUrl: Url{ + wantUrl: GitUrl{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -84,12 +98,12 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should fail with only GitHub host", url: "https://github.com/", - wantErr: "url path should contain /*", + wantErr: missingUserAndRepoError, }, { name: "should parse GitHub repo with file path", url: "https://github.com/devfile/library/blob/main/devfile.yaml", - wantUrl: Url{ + wantUrl: GitUrl{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -102,7 +116,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse GitHub repo with raw file path", url: "https://raw.githubusercontent.com/devfile/library/main/devfile.yaml", - wantUrl: Url{ + wantUrl: GitUrl{ Protocol: "https", Host: "raw.githubusercontent.com", Owner: "devfile", @@ -115,38 +129,38 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should fail with missing GitHub repo", url: "https://github.com/devfile", - wantErr: "url path should contain /*", + wantErr: missingUserAndRepoError, }, { name: "should fail with missing GitHub blob", url: "https://github.com/devfile/library/main/devfile.yaml", - wantErr: "url path to directory or file should contain*", + wantErr: invalidUrlPathError, }, { name: "should fail with missing GitHub tree", url: "https://github.com/devfile/library/main/tests/yamls", - wantErr: "url path to directory or file should contain*", + wantErr: invalidUrlPathError, }, { name: "should fail with just GitHub tree", url: "https://github.com/devfile/library/tree", - wantErr: "url path should contain ////*", + wantErr: invalidGitHubPathError, }, { name: "should fail with just GitHub blob", url: "https://github.com/devfile/library/blob", - wantErr: "url path should contain ////*", + wantErr: invalidGitHubPathError, }, { name: "should fail with invalid GitHub raw file path", url: "https://raw.githubusercontent.com/devfile/library/devfile.yaml", - wantErr: "raw url path should contain ///*", + wantErr: invalidGitHubRawPathError, }, // Gitlab { name: "should parse GitLab repo with root path", url: "https://gitlab.com/gitlab-org/gitlab-foss", - wantUrl: Url{ + wantUrl: GitUrl{ Protocol: "https", Host: "gitlab.com", Owner: "gitlab-org", @@ -159,12 +173,12 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should fail with only GitLab host", url: "https://gitlab.com/", - wantErr: "url path should contain /*", + wantErr: missingUserAndRepoError, }, { name: "should parse GitLab repo with file path", url: "https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/README.md", - wantUrl: Url{ + wantUrl: GitUrl{ Protocol: "https", Host: "gitlab.com", Owner: "gitlab-org", @@ -177,23 +191,23 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should fail with missing GitLab repo", url: "https://gitlab.com/gitlab-org", - wantErr: "url path should contain /*", + wantErr: missingUserAndRepoError, }, { name: "should fail with missing GitLab keywords", url: "https://gitlab.com/gitlab-org/gitlab-foss/-/master/directory/README.md", - wantErr: "url path should contain 'blob' or 'tree' or 'raw'*", + wantErr: missingGitLabKeywordError, }, { name: "should fail with missing GitLab file or directory path", url: "https://gitlab.com/gitlab-org/gitlab-foss/-/tree/master", - wantErr: "url path to directory or file should contain //*", + wantErr: invalidGitLabPathError, }, // Bitbucket { name: "should parse Bitbucket repo with root path", url: "https://bitbucket.org/fake-owner/fake-public-repo", - wantUrl: Url{ + wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -206,12 +220,12 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should fail with only Bitbucket host", url: "https://bitbucket.org/", - wantErr: "url path should contain /*", + wantErr: missingUserAndRepoError, }, { name: "should parse Bitbucket repo with file path", url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/README.md", - wantUrl: Url{ + wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -224,7 +238,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse Bitbucket file path with nested path", url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/directory/test.txt", - wantUrl: Url{ + wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -237,7 +251,7 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should parse Bitbucket repo with raw file path", url: "https://bitbucket.org/fake-owner/fake-public-repo/raw/main/README.md", - wantUrl: Url{ + wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", Owner: "fake-owner", @@ -250,17 +264,17 @@ func Test_ParseGitUrl(t *testing.T) { { name: "should fail with missing Bitbucket repo", url: "https://bitbucket.org/fake-owner", - wantErr: "url path should contain /*", + wantErr: missingUserAndRepoError, }, { name: "should fail with invalid Bitbucket directory or file path", url: "https://bitbucket.org/fake-owner/fake-public-repo/main/README.md", - wantErr: "url path should contain path to directory or file*", + wantErr: invalidBitbucketPathError, }, { name: "should fail with missing Bitbucket keywords", url: "https://bitbucket.org/fake-owner/fake-public-repo/main/test/README.md", - wantErr: "url path should contain 'raw' or 'src'*", + wantErr: missingBitbucketKeywordError, }, } @@ -281,12 +295,12 @@ func Test_ParseGitUrl(t *testing.T) { func Test_GetGitRawFileAPI(t *testing.T) { tests := []struct { name string - g Url + g GitUrl want string }{ { name: "Github url", - g: Url{ + g: GitUrl{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -298,19 +312,19 @@ func Test_GetGitRawFileAPI(t *testing.T) { }, { name: "GitLab url", - g: Url{ + g: GitUrl{ Protocol: "https", Host: "gitlab.com", Owner: "gitlab-org", Repo: "gitlab", - Revision: "master", + Revision: "v15.11.0-ee", Path: "README.md", }, - want: "https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab/repository/files/README.md/raw", + want: "https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab/repository/files/README.md/raw?ref=v15.11.0-ee", }, { name: "Bitbucket url", - g: Url{ + g: GitUrl{ Protocol: "https", Host: "bitbucket.org", Owner: "owner", @@ -322,7 +336,7 @@ func Test_GetGitRawFileAPI(t *testing.T) { }, { name: "Empty GitUrl", - g: Url{}, + g: GitUrl{}, want: "", }, } @@ -338,7 +352,7 @@ func Test_GetGitRawFileAPI(t *testing.T) { } func Test_IsPublic(t *testing.T) { - publicGitUrl := Url{ + publicGitUrl := GitUrl{ Protocol: "https", Host: "github.com", Owner: "devfile", @@ -347,7 +361,7 @@ func Test_IsPublic(t *testing.T) { token: "fake-token", } - privateGitUrl := Url{ + privateGitUrl := GitUrl{ Protocol: "https", Host: "github.com", Owner: "not", @@ -360,7 +374,7 @@ func Test_IsPublic(t *testing.T) { tests := []struct { name string - g Url + g GitUrl want bool }{ { @@ -387,11 +401,8 @@ func Test_IsPublic(t *testing.T) { func Test_CloneGitRepo(t *testing.T) { tempInvalidDir := t.TempDir() - tempDirGitHub := t.TempDir() - tempDirGitLab := t.TempDir() - tempDirBitbucket := t.TempDir() - invalidGitUrl := Url{ + invalidGitUrl := GitUrl{ Protocol: "", Host: "", Owner: "nonexistent", @@ -399,31 +410,7 @@ func Test_CloneGitRepo(t *testing.T) { Revision: "nonexistent", } - validPublicGitHubUrl := Url{ - Protocol: "https", - Host: "github.com", - Owner: "devfile", - Repo: "library", - Revision: "main", - } - - validPublicGitLabUrl := Url{ - Protocol: "https", - Host: "gitlab.com", - Owner: "mike-hoang", - Repo: "public-testing-repo", - Revision: "main", - } - - validPublicBitbucketUrl := Url{ - Protocol: "https", - Host: "bitbucket.org", - Owner: "mike-hoang", - Repo: "public-testing-repo", - Revision: "master", - } - - invalidPrivateGitHubRepo := Url{ + invalidPrivateGitHubRepo := GitUrl{ Protocol: "https", Host: "github.com", Owner: "fake-owner", @@ -438,7 +425,7 @@ func Test_CloneGitRepo(t *testing.T) { tests := []struct { name string - gitUrl Url + gitUrl GitUrl destDir string wantErr string }{ @@ -460,21 +447,6 @@ func Test_CloneGitRepo(t *testing.T) { destDir: tempInvalidDir, wantErr: privateRepoBadTokenErr, }, - { - name: "should be able to clone valid public github url", - gitUrl: validPublicGitHubUrl, - destDir: tempDirGitHub, - }, - { - name: "should be able to clone valid public gitlab url", - gitUrl: validPublicGitLabUrl, - destDir: tempDirGitLab, - }, - { - name: "should be able to clone valid public bitbucket url", - gitUrl: validPublicBitbucketUrl, - destDir: tempDirBitbucket, - }, } for _, tt := range tests { diff --git a/pkg/git/mock.go b/pkg/git/mock.go index a6b6c77d..458ab369 100644 --- a/pkg/git/mock.go +++ b/pkg/git/mock.go @@ -44,30 +44,43 @@ var mockExecute = func(baseDir string, cmd CommandType, args ...string) ([]byte, u, _ := url.Parse(args[1]) password, hasPassword := u.User.Password() + resourceFile, err := os.Create(filepath.Clean(baseDir) + "/resource.file") + if err != nil { + return nil, fmt.Errorf("failed to create test resource: %v", err) + } + // private repository if hasPassword { switch password { case "valid-token": - _, err := os.Create(filepath.Clean(baseDir) + "/private-repo-resource.txt") + _, err := resourceFile.WriteString("private repo\n") if err != nil { - return nil, fmt.Errorf("failed to create test resource: %v", err) + return nil, fmt.Errorf("failed to write to test resource: %v", err) } - return []byte("test"), nil + return []byte(""), nil default: return []byte(""), fmt.Errorf("not a valid token") } } - // public repository - _, err := os.Create(filepath.Clean(baseDir) + "/public-repo-resource.txt") + + _, err = resourceFile.WriteString("public repo\n") if err != nil { - return nil, fmt.Errorf("failed to create test resource: %v", err) + return nil, fmt.Errorf("failed to write to test resource: %v", err) } - return []byte("test"), nil + return []byte(""), nil } if len(args) > 0 && args[0] == "switch" { revision := strings.TrimPrefix(args[2], "origin/") if revision != "invalid-revision" { + resourceFile, err := os.OpenFile(filepath.Clean(baseDir)+"/resource.file", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return nil, fmt.Errorf("failed to open test resource: %v", err) + } + _, err = resourceFile.WriteString("git switched") + if err != nil { + return nil, fmt.Errorf("failed to write to test resource: %v", err) + } return []byte("git switched to revision"), nil } return []byte(""), fmt.Errorf("failed to switch revision") diff --git a/pkg/util/util.go b/pkg/util/util.go index aa490a30..b4a4fb3c 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1100,7 +1100,7 @@ func DownloadInMemory(params HTTPRequestParams) ([]byte, error) { ResponseHeaderTimeout: HTTPRequestResponseTimeout, }, Timeout: HTTPRequestResponseTimeout} - var g git.Url + var g git.GitUrl var err error if IsGitProviderRepo(params.URL) { @@ -1113,7 +1113,7 @@ func DownloadInMemory(params HTTPRequestParams) ([]byte, error) { return downloadInMemoryWithClient(params, httpClient, g) } -func downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient, g git.Url) ([]byte, error) { +func downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient, g git.GitUrl) ([]byte, error) { var url string url = params.URL req, err := http.NewRequest("GET", url, nil) @@ -1128,6 +1128,7 @@ func downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient, return nil, err } if !g.IsPublic(params.Timeout) { + // check that the token is valid before adding to the header err = g.SetToken(params.Token, params.Timeout) if err != nil { return nil, err