updating tests

Signed-off-by: Michael Hoang <[email protected]>

Author: mike-hoang

README.md

@@ -12,8 +12,9 @@ The Devfile Parser library is a Golang module that:
 2. writes to the devfile.yaml with the updated data.
 3. generates Kubernetes objects for the various devfile resources.
 4. defines util functions for the devfile.
+5. downloads resources from a parent devfile if specified in the devfile.yaml
 
-## Private Repository Support
+## Private repository support
 
 Tokens are required to be set in the following cases:
 1. parsing a devfile from a private repository
@@ -24,19 +25,28 @@ Set the token for the repository:
 ```go
 parser.ParserArgs{
 	...
-	URL: <url-to-devfile-on-supported-git-provider>
+	// URL must point to a devfile.yaml
+	URL: <url-to-devfile-on-supported-git-provider-repo>/devfile.yaml
 	Token: <repo-personal-access-token>
 	...
 }
 ```
 Note: The url must also be set with a supported git provider repo url.
 
+Minimum token scope required:
+1. GitHub: Read access to code
+2. GitLab: Read repository
+3. Bitbucket: Read repository
+
+Note: To select token scopes for GitHub, a fine-grained token is required.
+
 For more information about personal access tokens:
 1. [GitHub docs](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)
 2. [GitLab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token)
 3. [Bitbucket docs](https://support.atlassian.com/bitbucket-cloud/docs/repository-access-tokens/)
 
 [1] Currently, this works under the assumption that the token can authenticate the devfile and the parent devfile; both devfiles are in the same repository.
+
 [2] In this scenario, the token will be used to authenticate the main devfile.
 
 ## Usage
@@ -199,6 +209,15 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
    }
    ```
 
+9. When parsing a devfile that contains a parent reference, if the parent uri is a supported git provider repo url with the correct personal access token, all resources from the parent git repo excluding the parent devfile.yaml will be downloaded to the location of the devfile being parsed. **Note: The URL must point to a devfile.yaml**
+   ```yaml
+   schemaVersion: 2.2.0
+   ...
+   parent:
+      uri: <uri-to-parent-devfile>/devfile.yaml
+   ...
+   ```
+
 ## Projects using devfile/library
 
 The following projects are consuming this library as a Golang dependency

pkg/devfile/parser/context/context.go

@@ -73,14 +73,6 @@ func NewURLDevfileCtx(url string) DevfileCtx {
 	}
 }
 
-// NewPrivateURLDevfileCtx returns a new DevfileCtx type object
-func NewPrivateURLDevfileCtx(url string, token string) DevfileCtx {
-	return DevfileCtx{
-		url:   url,
-		token: token,
-	}
-}
-
 // NewByteContentDevfileCtx set devfile content from byte data and returns a new DevfileCtx type object and error
 func NewByteContentDevfileCtx(data []byte) (d DevfileCtx, err error) {
 	err = d.SetDevfileContentFromBytes(data)
@@ -166,6 +158,11 @@ func (d *DevfileCtx) GetToken() string {
 	return d.token
 }
 
+// SetToken sets the token for the devfile
+func (d *DevfileCtx) SetToken(token string) {
+	d.token = token
+}
+
 // SetAbsPath sets absolute file path for devfile
 func (d *DevfileCtx) SetAbsPath() (err error) {
 	// Set devfile absolute path

pkg/devfile/parser/context/context_test.go

@@ -88,16 +88,12 @@ func TestNewURLDevfileCtx(t *testing.T) {
 		token = "fake-token"
 		url   = "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml"
 	)
-
-	{
-		d := NewPrivateURLDevfileCtx(url, token)
-		assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL())
-		assert.Equal(t, "fake-token", d.GetToken())
-	}
 	{
 		d := NewURLDevfileCtx(url)
 		assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL())
 		assert.Equal(t, "", d.GetToken())
+		d.SetToken(token)
+		assert.Equal(t, "fake-token", d.GetToken())
 	}
 }
 

pkg/devfile/parser/parse.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/devfile/library/v2/pkg/git"
+	"github.com/hashicorp/go-multierror"
 	"io/ioutil"
 	"net/url"
 	"os"
@@ -49,34 +50,49 @@ import (
 
 // downloadGitRepoResources is exposed as a global variable for the purpose of running mock tests
 var downloadGitRepoResources = func(url string, destDir string, httpTimeout *int, token string) error {
+	var returnedErr error
+
 	gitUrl, err := git.NewGitUrlWithURL(url)
 	if err != nil {
 		return err
 	}
 
-	if gitUrl.IsGitProviderRepo() && gitUrl.IsFile {
-		stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources"))
+	if gitUrl.IsGitProviderRepo() {
+		if !gitUrl.IsFile || gitUrl.Revision == "" || !strings.Contains(gitUrl.Path, OutputDevfileYamlPath) {
+			return fmt.Errorf("error getting devfile from url: failed to retrieve %s", url)
+		}
+
+		stackDir, err := os.MkdirTemp("", fmt.Sprintf("git-resources"))
 		if err != nil {
 			return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err)
 		}
-		defer os.RemoveAll(stackDir)
+
+		defer func(path string) {
+			err := os.RemoveAll(path)
+			if err != nil {
+				returnedErr = multierror.Append(returnedErr, err)
+			}
+		}(stackDir)
 
 		if !gitUrl.IsPublic(httpTimeout) {
 			err = gitUrl.SetToken(token, httpTimeout)
 			if err != nil {
-				return err
+				returnedErr = multierror.Append(returnedErr, err)
+				return returnedErr
 			}
 		}
 
 		err = gitUrl.CloneGitRepo(stackDir)
 		if err != nil {
-			return err
+			returnedErr = multierror.Append(returnedErr, err)
+			return returnedErr
 		}
 
 		dir := path.Dir(path.Join(stackDir, gitUrl.Path))
 		err = git.CopyAllDirFiles(dir, destDir)
 		if err != nil {
-			return err
+			returnedErr = multierror.Append(returnedErr, err)
+			return returnedErr
 		}
 	}
 
@@ -163,15 +179,15 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) {
 	} else if args.Path != "" {
 		d.Ctx = devfileCtx.NewDevfileCtx(args.Path)
 	} else if args.URL != "" {
-		if args.Token != "" {
-			d.Ctx = devfileCtx.NewPrivateURLDevfileCtx(args.URL, args.Token)
-		} else {
-			d.Ctx = devfileCtx.NewURLDevfileCtx(args.URL)
-		}
+		d.Ctx = devfileCtx.NewURLDevfileCtx(args.URL)
 	} else {
 		return d, errors.Wrap(err, "the devfile source is not provided")
 	}
 
+	if args.Token != "" {
+		d.Ctx.SetToken(args.Token)
+	}
+
 	tool := resolverTools{
 		defaultNamespace: args.DefaultNamespace,
 		registryURLs:     args.RegistryURLs,
@@ -475,10 +491,9 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D
 		}
 
 		token := curDevfileCtx.GetToken()
+		d.Ctx = devfileCtx.NewURLDevfileCtx(newUri)
 		if token != "" {
-			d.Ctx = devfileCtx.NewPrivateURLDevfileCtx(newUri, token)
-		} else {
-			d.Ctx = devfileCtx.NewURLDevfileCtx(newUri)
+			d.Ctx.SetToken(token)
 		}
 
 		destDir := path.Dir(curDevfileCtx.GetAbsPath())