adding support for public and private git providers

README.md

@@ -13,6 +13,25 @@ The Devfile Parser library is a Golang module that:
 3. generates Kubernetes objects for the various devfile resources.
 4. defines util functions for the devfile.
 
+## Private Repository Support
+
+Tokens are required to be set in the following cases:
+1. tooling client calling the library API
+2. parsing a devfile from a private repository
+3. parsing a devfile containing a parent devfile from a private repository
+
+Set the environment variables for the necessary git providers:
+   ```shell
+   export GITHUB_TOKEN=<account_token>
+   export GITLAB_TOKEN=<account_token>
+   export BITBUCKET_TOKEN=<account_token>
+   ```
+
+For more information about personal access tokens:
+1. [GitHub docs](https://docs.github.com/en/[email protected]/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)
+2. [GitLab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token)
+3. [Bitbucket docs](https://support.atlassian.com/bitbucket-cloud/docs/repository-access-tokens/)
+
 ## Usage
 
 The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/github.com/devfile/library). 
@@ -169,19 +188,17 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
 
 8. To get resources from Git repositories
    ```go
+   // Supported url formats
+   url = "https://github.com/<owner>/<repo name>"
+   url = "https://bitbucket.org/<owner>/<repo name>"
+   url = "https://gitlab.com/<owner>/<repo name>"
+
    // Parse the repo url
    gitUrl, err := util.ParseGitUrl(url)
 
    // Clone the repo to a destination dir
    err = util.CloneGitRepo(gitUrl, destDir)
    ```
-   If repository is private, set the correct environment variables
-   ```shell
-   # credentials for private repositories
-   export GITHUB_TOKEN=<account_token>
-   export GITLAB_TOKEN=<account_token>
-   export BITBUCKET_TOKEN=<account_token>
-   ```
 
 ## Projects using devfile/library
 

pkg/devfile/parser/parse.go

@@ -425,14 +425,15 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D
 		}
 
 		d.Ctx = devfileCtx.NewURLDevfileCtx(newUri)
-		if strings.Contains(newUri, util.RawGitHubHost) {
+		if strings.Contains(newUri, util.RawGitHubHost) || strings.Contains(newUri, util.GitHubHost) ||
+			strings.Contains(newUri, util.GitLabHost) || strings.Contains(newUri, util.BitbucketToken) {
 			gitUrl, err := util.ParseGitUrl(newUri)
 			if err != nil {
 				return DevfileObj{}, err
 			}
 			if gitUrl.IsFile {
 				destDir := path.Dir(curDevfileCtx.GetAbsPath())
-				err = getResourcesFromGit(gitUrl, destDir)
+				err = getResourcesFromGit(gitUrl, destDir, tool.httpTimeout)
 				if err != nil {
 					return DevfileObj{}, err
 				}
@@ -445,14 +446,14 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D
 	return populateAndParseDevfile(d, newResolveCtx, tool, true)
 }
 
-func getResourcesFromGit(g util.GitUrl, destDir string) error {
+func getResourcesFromGit(g util.GitUrl, destDir string, httpTimeout *int) error {
 	stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources"))
 	if err != nil {
 		return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err)
 	}
 	defer os.RemoveAll(stackDir)
 
-	err = util.CloneGitRepo(g, stackDir)
+	err = util.CloneGitRepo(g, stackDir, httpTimeout)
 	if err != nil {
 		return err
 	}

pkg/devfile/parser/parse_test.go

@@ -4166,6 +4166,8 @@ func Test_getResourcesFromGit(t *testing.T) {
 	}
 	defer os.RemoveAll(destDir)
 
+	httpTimeout := 0
+
 	invalidGitHubUrl := util.GitUrl{
 		Protocol: "",
 		Host:     "",
@@ -4205,7 +4207,7 @@ func Test_getResourcesFromGit(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := getResourcesFromGit(tt.gitUrl, tt.destDir)
+			err := getResourcesFromGit(tt.gitUrl, tt.destDir, &httpTimeout)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err)
 			}

pkg/util/git.go

@@ -201,28 +201,68 @@ func (g *GitUrl) parseBitbucketUrl(url *url.URL) error {
 	return err
 }
 
+// ValidateToken makes a http get request to the repo with the GitUrl token
+// Returns an error if the get request fails
+// If token is empty or invalid and validate succeeds, the repository is public
+func (g *GitUrl) ValidateToken(params HTTPRequestParams) error {
+	var apiUrl string
+
+	switch g.Host {
+	case GitHubHost, RawGitHubHost:
+		apiUrl = fmt.Sprintf("https://api.github.com/repos/%s/%s", g.Owner, g.Repo)
+	case GitLabHost:
+		apiUrl = fmt.Sprintf("https://gitlab.com/api/v4/projects/%s%%2F%s", g.Owner, g.Repo)
+	case BitbucketHost:
+		apiUrl = fmt.Sprintf("https://api.bitbucket.org/2.0/repositories/%s/%s", g.Owner, g.Repo)
+	default:
+		apiUrl = fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, g.Host, g.Owner, g.Repo)
+	}
+
+	params.URL = apiUrl
+	res, err := HTTPGetRequest(params, 0)
+	if len(res) == 0 || err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // CloneGitRepo clones a git repo to a destination directory
-func CloneGitRepo(g GitUrl, destDir string) error {
+// Only supports git repositories hosted on GitHub, GitLab, and Bitbucket
+func CloneGitRepo(g GitUrl, destDir string, httpTimeout *int) error {
+	exist := CheckPathExists(destDir)
+	if !exist {
+		return fmt.Errorf("failed to clone repo, destination directory: '%s' does not exists", destDir)
+	}
+
 	host := g.Host
 	if host == RawGitHubHost {
 		host = GitHubHost
 	}
 
-	isPublic := true
 	repoUrl := fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, host, g.Owner, g.Repo)
 
 	params := HTTPRequestParams{
-		URL: repoUrl,
+		Timeout: httpTimeout,
 	}
 
-	// check if the git repo is public
-	_, err := HTTPGetRequest(params, 0)
+	// public repos will succeed even if token is invalid or empty
+	err := g.ValidateToken(params)
+
 	if err != nil {
-		// private git repo requires authentication
-		isPublic = false
-		repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo)
-		if g.Host == BitbucketHost {
-			repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo)
+		if g.token != "" {
+			params.Token = g.token
+			err := g.ValidateToken(params)
+			if err != nil {
+				return fmt.Errorf("failed to validate git url with token, ensure that the url and token is correct. error: %v", err)
+			} else {
+				repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo)
+				if g.Host == BitbucketHost {
+					repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo)
+				}
+			}
+		} else {
+			return fmt.Errorf("failed to validate git url without a token, ensure that a token is set if the repo is private. error: %v", err)
 		}
 	}
 
@@ -236,13 +276,7 @@ func CloneGitRepo(g GitUrl, destDir string) error {
 
 	_, err = c.CombinedOutput()
 	if err != nil {
-		if !isPublic {
-			if g.token == "" {
-				return fmt.Errorf("failed to clone repo without a token, ensure that a token is set if the repo is private. error: %v", err)
-			}
-			return fmt.Errorf("failed to clone repo with token, ensure that the url and token is correct. error: %v", err)
-		}
-		return fmt.Errorf("failed to clone repo, ensure that the url is correct. error: %v", err)
+		return fmt.Errorf("failed to clone repo, ensure that the git url is correct. error: %v", err)
 	}
 
 	return nil

pkg/util/git_test.go

@@ -355,10 +355,13 @@ func Test_ParseGitUrl(t *testing.T) {
 }
 
 func TestCloneGitRepo(t *testing.T) {
+	tempInvalidDir := t.TempDir()
 	tempDirGitHub := t.TempDir()
 	tempDirGitLab := t.TempDir()
 	tempDirBitbucket := t.TempDir()
 
+	httpTimeout := 0
+
 	invalidGitUrl := GitUrl{
 		Protocol: "",
 		Host:     "",
@@ -367,31 +370,31 @@ func TestCloneGitRepo(t *testing.T) {
 		Branch:   "nonexistent",
 	}
 
-	validGitHubUrl := GitUrl{
+	validPublicGitHubUrl := GitUrl{
 		Protocol: "https",
 		Host:     "github.com",
 		Owner:    "devfile",
 		Repo:     "library",
 		Branch:   "main",
 	}
 
-	validGitLabUrl := GitUrl{
+	validPublicGitLabUrl := GitUrl{
 		Protocol: "https",
 		Host:     "gitlab.com",
 		Owner:    "mike-hoang",
 		Repo:     "public-testing-repo",
 		Branch:   "main",
 	}
 
-	validBitbucketUrl := GitUrl{
+	validPublicBitbucketUrl := GitUrl{
 		Protocol: "https",
 		Host:     "bitbucket.org",
 		Owner:    "mike-hoang",
 		Repo:     "public-testing-repo",
 		Branch:   "master",
 	}
 
-	privateGitHubRepo := GitUrl{
+	invalidPrivateGitHubRepo := GitUrl{
 		Protocol: "https",
 		Host:     "github.com",
 		Owner:    "fake-owner",
@@ -400,8 +403,9 @@ func TestCloneGitRepo(t *testing.T) {
 		token:    "fake-github-token",
 	}
 
-	privateRepoMissingTokenErr := "failed to clone repo without a token*"
-	privateRepoBadTokenErr := "failed to clone repo with token*"
+	privateRepoBadTokenErr := "failed to validate git url with token*"
+	publicRepoInvalidUrlErr := "failed to validate git url without a token"
+	missingDestDirErr := "failed to clone repo, destination directory*"
 
 	tests := []struct {
 		name    string
@@ -410,37 +414,43 @@ func TestCloneGitRepo(t *testing.T) {
 		wantErr string
 	}{
 		{
-			name:    "should fail with invalid git url",
+			name:    "should fail with invalid destination directory",
 			gitUrl:  invalidGitUrl,
 			destDir: filepath.Join(os.TempDir(), "nonexistent"),
-			wantErr: privateRepoMissingTokenErr,
+			wantErr: missingDestDirErr,
 		},
 		{
-			name:    "should fail to clone valid private git url with a bad token",
-			gitUrl:  privateGitHubRepo,
-			destDir: filepath.Join(os.TempDir(), "nonexistent"),
+			name:    "should fail with invalid git url",
+			gitUrl:  invalidGitUrl,
+			destDir: tempInvalidDir,
+			wantErr: publicRepoInvalidUrlErr,
+		},
+		{
+			name:    "should fail to clone invalid private git url with a bad token",
+			gitUrl:  invalidPrivateGitHubRepo,
+			destDir: tempInvalidDir,
 			wantErr: privateRepoBadTokenErr,
 		},
 		{
 			name:    "should be able to clone valid public github url",
-			gitUrl:  validGitHubUrl,
+			gitUrl:  validPublicGitHubUrl,
 			destDir: tempDirGitHub,
 		},
 		{
 			name:    "should be able to clone valid public gitlab url",
-			gitUrl:  validGitLabUrl,
+			gitUrl:  validPublicGitLabUrl,
 			destDir: tempDirGitLab,
 		},
 		{
 			name:    "should be able to clone valid public bitbucket url",
-			gitUrl:  validBitbucketUrl,
+			gitUrl:  validPublicBitbucketUrl,
 			destDir: tempDirBitbucket,
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := CloneGitRepo(tt.gitUrl, tt.destDir)
+			err := CloneGitRepo(tt.gitUrl, tt.destDir, &httpTimeout)
 			if (err != nil) != (tt.wantErr != "") {
 				t.Errorf("Unxpected error: %t, want: %v", err, tt.wantErr)
 			} else if err != nil {