wip: set token outside of cloning function

Signed-off-by: Michael Hoang <[email protected]>

Author: mike-hoang

README.md

@@ -18,20 +18,24 @@ The Devfile Parser library is a Golang module that:
 Tokens are required to be set in the following cases:
 1. tooling client calling the library API
 2. parsing a devfile from a private repository
-3. parsing a devfile containing a parent devfile from a private repository
-
-Set the environment variables for the necessary git providers:
-   ```shell
-   export GITHUB_TOKEN=<account_token>
-   export GITLAB_TOKEN=<account_token>
-   export BITBUCKET_TOKEN=<account_token>
-   ```
+3. parsing a devfile containing a parent devfile from a private repository [1]
+
+Set the token for the repository:
+```go
+parser.ParserArgs{
+	...
+	Token: <repo-personal-access-token>
+	...
+}
+```
 
 For more information about personal access tokens:
 1. [GitHub docs](https://docs.github.com/en/[email protected]/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)
 2. [GitLab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token)
 3. [Bitbucket docs](https://support.atlassian.com/bitbucket-cloud/docs/repository-access-tokens/)
 
+[1] currently, this works under the assumption that the token can authenticate the devfile and the parent devfile; both devfiles are in the same repository.
+
 ## Usage
 
 The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/github.com/devfile/library). 

pkg/devfile/parser/parse.go

@@ -97,6 +97,8 @@ type ParserArgs struct {
 	// RegistryURLs is a list of registry hosts which parser should pull parent devfile from.
 	// If registryUrl is defined in devfile, this list will be ignored.
 	RegistryURLs []string
+	// Token is a GitHub, GitLab, or Bitbucket personal access token used with a private git repo uri
+	Token string
 	// DefaultNamespace is the default namespace to use
 	// If namespace is defined under devfile's parent kubernetes object, this namespace will be ignored.
 	DefaultNamespace string
@@ -132,6 +134,7 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) {
 		context:          args.Context,
 		k8sClient:        args.K8sClient,
 		httpTimeout:      args.HTTPTimeout,
+		token:            args.Token,
 	}
 
 	flattenedDevfile := true
@@ -175,6 +178,8 @@ type resolverTools struct {
 	// RegistryURLs is a list of registry hosts which parser should pull parent devfile from.
 	// If registryUrl is defined in devfile, this list will be ignored.
 	registryURLs []string
+	// Token is a GitHub, GitLab, or Bitbucket personal access token used with a private git repo uri
+	token string
 	// Context is the context used for making Kubernetes or HTTP requests
 	context context.Context
 	// K8sClient is the Kubernetes client instance used for interacting with a cluster
@@ -425,15 +430,14 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D
 		}
 
 		d.Ctx = devfileCtx.NewURLDevfileCtx(newUri)
-		if strings.Contains(newUri, util.RawGitHubHost) || strings.Contains(newUri, util.GitHubHost) ||
-			strings.Contains(newUri, util.GitLabHost) || strings.Contains(newUri, util.BitbucketToken) {
+		if util.IsGitProviderRepo(newUri) {
 			gitUrl, err := util.ParseGitUrl(newUri)
 			if err != nil {
 				return DevfileObj{}, err
 			}
 			if gitUrl.IsFile {
 				destDir := path.Dir(curDevfileCtx.GetAbsPath())
-				err = getResourcesFromGit(gitUrl, destDir, tool.httpTimeout)
+				err = getResourcesFromGit(gitUrl, destDir, tool.httpTimeout, tool.token)
 				if err != nil {
 					return DevfileObj{}, err
 				}
@@ -446,14 +450,21 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D
 	return populateAndParseDevfile(d, newResolveCtx, tool, true)
 }
 
-func getResourcesFromGit(g util.GitUrl, destDir string, httpTimeout *int) error {
+func getResourcesFromGit(g util.GitUrl, destDir string, httpTimeout *int, repoToken string) error {
 	stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources"))
 	if err != nil {
 		return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err)
 	}
 	defer os.RemoveAll(stackDir)
 
-	err = util.CloneGitRepo(g, stackDir, httpTimeout)
+	if !g.IsPublic(httpTimeout) {
+		err = g.SetToken(repoToken, httpTimeout)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = util.CloneGitRepo(g, stackDir)
 	if err != nil {
 		return err
 	}

pkg/devfile/parser/parse_test.go

@@ -4207,7 +4207,7 @@ func Test_getResourcesFromGit(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := getResourcesFromGit(tt.gitUrl, tt.destDir, &httpTimeout)
+			err := getResourcesFromGit(tt.gitUrl, tt.destDir, &httpTimeout, "")
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err)
 			}

pkg/util/git.go

@@ -29,10 +29,6 @@ const (
 	RawGitHubHost string = "raw.githubusercontent.com"
 	GitLabHost    string = "gitlab.com"
 	BitbucketHost string = "bitbucket.org"
-
-	GitHubToken    string = "GITHUB_TOKEN"
-	GitLabToken    string = "GITLAB_TOKEN"
-	BitbucketToken string = "BITBUCKET_TOKEN"
 )
 
 type GitUrl struct {
@@ -46,7 +42,8 @@ type GitUrl struct {
 	IsFile   bool   // defines if the URL points to a file in the repo
 }
 
-// ParseGitUrl extracts information from a GitHub, GitLab, or Bitbucket url
+// ParseGitUrl extracts information from a support git url
+// Only supports git repositories hosted on GitHub, GitLab, and Bitbucket
 func ParseGitUrl(fullUrl string) (GitUrl, error) {
 	var g GitUrl
 
@@ -83,7 +80,6 @@ func (g *GitUrl) parseGitHubUrl(url *url.URL) error {
 
 	g.Protocol = url.Scheme
 	g.Host = url.Host
-	g.token = os.Getenv(GitHubToken)
 
 	if g.Host == RawGitHubHost {
 		g.IsFile = true
@@ -130,7 +126,6 @@ func (g *GitUrl) parseGitLabUrl(url *url.URL) error {
 	g.Protocol = url.Scheme
 	g.Host = url.Host
 	g.IsFile = false
-	g.token = os.Getenv(GitLabToken)
 
 	// GitLab urls contain a '-' separating the root of the repo
 	// and the path to a file or directory
@@ -171,7 +166,6 @@ func (g *GitUrl) parseBitbucketUrl(url *url.URL) error {
 	g.Protocol = url.Scheme
 	g.Host = url.Host
 	g.IsFile = false
-	g.token = os.Getenv(BitbucketToken)
 
 	splitUrl = strings.SplitN(url.Path[1:], "/", 5)
 	if len(splitUrl) < 2 {
@@ -201,10 +195,31 @@ func (g *GitUrl) parseBitbucketUrl(url *url.URL) error {
 	return err
 }
 
-// ValidateToken makes a http get request to the repo with the GitUrl token
+// SetToken validates the token with a get request to the repo before setting the token
+// Defaults token to empty on failure.
+func (g *GitUrl) SetToken(token string, httpTimeout *int) error {
+	err := g.validateToken(HTTPRequestParams{Token: token, Timeout: httpTimeout})
+	if err != nil {
+		g.token = ""
+		return fmt.Errorf("failed to set token. error: %v", err)
+	}
+	g.token = token
+	return nil
+}
+
+// IsPublic checks if the GitUrl is public with a get request to the repo using an empty token
+// Returns true if the request succeeds
+func (g *GitUrl) IsPublic(httpTimeout *int) bool {
+	err := g.validateToken(HTTPRequestParams{Token: "", Timeout: httpTimeout})
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+// validateToken makes a http get request to the repo with the GitUrl token
 // Returns an error if the get request fails
-// If token is empty or invalid and validate succeeds, the repository is public
-func (g *GitUrl) ValidateToken(params HTTPRequestParams) error {
+func (g *GitUrl) validateToken(params HTTPRequestParams) error {
 	var apiUrl string
 
 	switch g.Host {
@@ -227,9 +242,17 @@ func (g *GitUrl) ValidateToken(params HTTPRequestParams) error {
 	return nil
 }
 
-// CloneGitRepo clones a git repo to a destination directory
-// Only supports git repositories hosted on GitHub, GitLab, and Bitbucket
-func CloneGitRepo(g GitUrl, destDir string, httpTimeout *int) error {
+// IsGitProviderRepo checks if the url matches a repo from a supported git provider
+func IsGitProviderRepo(url string) bool {
+	if strings.Contains(url, RawGitHubHost) || strings.Contains(url, GitHubHost) ||
+		strings.Contains(url, GitLabHost) || strings.Contains(url, BitbucketHost) {
+		return true
+	}
+	return false
+}
+
+// CloneGitRepo clones a git repo to a destination directory (either an absolute or relative path)
+func CloneGitRepo(g GitUrl, destDir string) error {
 	exist := CheckPathExists(destDir)
 	if !exist {
 		return fmt.Errorf("failed to clone repo, destination directory: '%s' does not exists", destDir)
@@ -240,29 +263,13 @@ func CloneGitRepo(g GitUrl, destDir string, httpTimeout *int) error {
 		host = GitHubHost
 	}
 
-	repoUrl := fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, host, g.Owner, g.Repo)
-
-	params := HTTPRequestParams{
-		Timeout: httpTimeout,
-	}
-
-	// public repos will succeed even if token is invalid or empty
-	err := g.ValidateToken(params)
-
-	if err != nil {
-		if g.token != "" {
-			params.Token = g.token
-			err := g.ValidateToken(params)
-			if err != nil {
-				return fmt.Errorf("failed to validate git url with token, ensure that the url and token is correct. error: %v", err)
-			} else {
-				repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo)
-				if g.Host == BitbucketHost {
-					repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo)
-				}
-			}
-		} else {
-			return fmt.Errorf("failed to validate git url without a token, ensure that a token is set if the repo is private. error: %v", err)
+	var repoUrl string
+	if g.token == "" {
+		repoUrl = fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, host, g.Owner, g.Repo)
+	} else {
+		repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo)
+		if g.Host == BitbucketHost {
+			repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo)
 		}
 	}
 
@@ -274,9 +281,13 @@ func CloneGitRepo(g GitUrl, destDir string, httpTimeout *int) error {
 	c.Env = os.Environ()
 	c.Env = append(c.Env, "GIT_TERMINAL_PROMPT=0", "GIT_ASKPASS=/bin/echo")
 
-	_, err = c.CombinedOutput()
+	_, err := c.CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("failed to clone repo, ensure that the git url is correct. error: %v", err)
+		if g.token == "" {
+			return fmt.Errorf("failed to clone repo without a token, ensure that a token is set if the repo is private. error: %v", err)
+		} else {
+			return fmt.Errorf("failed to clone repo with token, ensure that the url and token is correct. error: %v", err)
+		}
 	}
 
 	return nil