OCI-based devifle registry support

go.mod

@@ -7,9 +7,10 @@ require (
 	github.com/Microsoft/go-winio v0.4.15 // indirect
 	github.com/Netflix/go-expect v0.0.0-20200312175327-da48e75238e2
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/containerd/containerd v1.3.3 // indirect
-	github.com/devfile/api/v2 v2.0.0-20210304212617-bfc3f501616b
-	github.com/devfile/library v1.0.0-alpha.2.0.20210323153322-3d708859f0b5
+	github.com/devfile/api/v2 v2.0.0-20210408144711-a313872749ed
+	github.com/devfile/library v1.0.0-alpha.2.0.20210409194304-7a52b221a48e
+	github.com/devfile/registry-support/index/generator v0.0.0-20210407161420-cd279527f873
+	github.com/devfile/registry-support/registry-library v0.0.0-20210407161420-cd279527f873
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
 	github.com/docker/go-connections v0.4.1-0.20200120150455-7dc0a2d6ddce
 	github.com/fatih/color v1.7.0
@@ -45,7 +46,7 @@ require (
 	github.com/redhat-developer/service-binding-operator v0.3.0
 	github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 // indirect
 	github.com/spf13/afero v1.2.2
-	github.com/spf13/cobra v0.0.5
+	github.com/spf13/cobra v1.1.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.6.1
 	github.com/tidwall/gjson v1.6.3

pkg/catalog/catalog.go

@@ -16,6 +16,8 @@ import (
 	"github.com/openshift/odo/pkg/log"
 	"github.com/openshift/odo/pkg/occlient"
 
+	indexSchema "github.com/devfile/registry-support/index/generator/schema"
+	registryLibrary "github.com/devfile/registry-support/registry-library/library"
 	registryUtil "github.com/openshift/odo/pkg/odo/cli/registry/util"
 	"github.com/openshift/odo/pkg/util"
 	olm "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
@@ -121,49 +123,56 @@ func convertURL(URL string) (string, error) {
 const indexPath = "/devfiles/index.json"
 
 // getRegistryDevfiles retrieves the registry's index devfile entries
-func getRegistryDevfiles(registry Registry) ([]DevfileComponentType, error) {
-	var devfileIndex []DevfileIndexEntry
+func getRegistryDevfiles(registry Registry) (registryDevfiles []DevfileComponentType, err error) {
+	var devfileIndex []indexSchema.Schema
 
-	URL, err := convertURL(registry.URL)
-	if err != nil {
-		return nil, errors.Wrapf(err, "Unable to convert URL %s", registry.URL)
-	}
-	registry.URL = URL
-	indexLink := registry.URL + indexPath
-	request := util.HTTPRequestParams{
-		URL: indexLink,
-	}
-	if registryUtil.IsSecure(registry.Name) {
-		token, err := keyring.Get(fmt.Sprintf("%s%s", util.CredentialPrefix, registry.Name), registryUtil.RegistryUser)
+	if strings.Contains(registry.URL, "github") {
+		// Github-based registry
+		URL, err := convertURL(registry.URL)
 		if err != nil {
-			return nil, errors.Wrap(err, "Unable to get secure registry credential from keyring")
+			return nil, errors.Wrapf(err, "Unable to convert URL %s", registry.URL)
+		}
+		registry.URL = URL
+		indexLink := registry.URL + indexPath
+		request := util.HTTPRequestParams{
+			URL: indexLink,
+		}
+		if registryUtil.IsSecure(registry.Name) {
+			token, err := keyring.Get(fmt.Sprintf("%s%s", util.CredentialPrefix, registry.Name), registryUtil.RegistryUser)
+			if err != nil {
+				return nil, errors.Wrap(err, "Unable to get secure registry credential from keyring")
+			}
+			request.Token = token
 		}
-		request.Token = token
-	}
 
-	cfg, err := preference.New()
-	if err != nil {
-		return nil, err
-	}
+		cfg, err := preference.New()
+		if err != nil {
+			return nil, err
+		}
 
-	jsonBytes, err := util.HTTPGetRequest(request, cfg.GetRegistryCacheTime())
-	if err != nil {
-		return nil, errors.Wrapf(err, "Unable to download the devfile index.json from %s", indexLink)
-	}
+		jsonBytes, err := util.HTTPGetRequest(request, cfg.GetRegistryCacheTime())
+		if err != nil {
+			return nil, errors.Wrapf(err, "Unable to download the devfile index.json from %s", indexLink)
+		}
 
-	err = json.Unmarshal(jsonBytes, &devfileIndex)
-	if err != nil {
-		return nil, errors.Wrapf(err, "Unable to unmarshal the devfile index.json from %s", indexLink)
+		err = json.Unmarshal(jsonBytes, &devfileIndex)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Unable to unmarshal the devfile index.json from %s", indexLink)
+		}
+	} else {
+		// OCI-based registry
+		devfileIndex, err = registryLibrary.GetRegistryStacks(registry.URL)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	var registryDevfiles []DevfileComponentType
-
 	for _, devfileIndexEntry := range devfileIndex {
 		stackDevfile := DevfileComponentType{
 			Name:        devfileIndexEntry.Name,
 			DisplayName: devfileIndexEntry.DisplayName,
 			Description: devfileIndexEntry.Description,
-			Link:        devfileIndexEntry.Links.Link,
+			Link:        devfileIndexEntry.Links["self"],
 			Registry:    registry,
 			Language:    devfileIndexEntry.Language,
 			Tags:        devfileIndexEntry.Tags,

pkg/catalog/catalog_test.go

@@ -184,7 +184,7 @@ OdoSettings:
   Experimental: true
   RegistryList:
   - Name: DefaultDevfileRegistry
-    URL: https://github.com/elsony/devfile-registry
+    URL: https://registry.devfile.io
   - Name: CheDevfileRegistry
     URL: https://che-devfile-registry.openshift.io/`,
 	))
@@ -211,7 +211,7 @@ OdoSettings:
 				},
 				{
 					Name:   "DefaultDevfileRegistry",
-					URL:    "https://github.com/elsony/devfile-registry",
+					URL:    "https://registry.devfile.io",
 					Secure: false,
 				},
 			},

pkg/catalog/types.go

@@ -30,22 +30,6 @@ type DevfileComponentType struct {
 	Tags        []string
 }
 
-// DevfileIndexEntry is the main struct of index.json from devfile registry
-type DevfileIndexEntry struct {
-	Name              string   `json:"name"`
-	DisplayName       string   `json:"displayName"`
-	Description       string   `json:"description"`
-	Supported         bool     `json:"supported"`
-	Tags              []string `json:"tags"`
-	Language          string   `json:"language"`
-	Icon              string   `json:"icon"`
-	GlobalMemoryLimit string   `json:"globalMemoryLimit"`
-	Registry          Registry `json:"registry"`
-	Links             struct {
-		Link string `json:"self"`
-	} `json:"links"`
-}
-
 // ComponentSpec is the spec for ComponentType
 type ComponentSpec struct {
 	AllTags         []string               `json:"allTags"`

pkg/devfile/adapters/kubernetes/utils/utils_test.go

@@ -974,7 +974,7 @@ func TestGetPreStartInitContainers(t *testing.T) {
 						t.Error(err)
 					}
 					err = devfileData.AddEvents(devfilev1.Events{
-						WorkspaceEvents: devfilev1.WorkspaceEvents{
+						DevWorkspaceEvents: devfilev1.DevWorkspaceEvents{
 							PreStart: tt.eventCommands,
 						},
 					})

pkg/devfile/validate/events_test.go

@@ -1,8 +1,9 @@
 package validate
 
 import (
-	"github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
 	"testing"
+
+	"github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
 )
 
 func Test_validateEvents(t *testing.T) {
@@ -15,7 +16,7 @@ func Test_validateEvents(t *testing.T) {
 		{
 			name: "just postStart event present",
 			events: v1alpha2.Events{
-				WorkspaceEvents: v1alpha2.WorkspaceEvents{
+				DevWorkspaceEvents: v1alpha2.DevWorkspaceEvents{
 					PostStart: []string{"asdf"},
 				},
 			},
@@ -24,7 +25,7 @@ func Test_validateEvents(t *testing.T) {
 		{
 			name: "preStart event present",
 			events: v1alpha2.Events{
-				WorkspaceEvents: v1alpha2.WorkspaceEvents{
+				DevWorkspaceEvents: v1alpha2.DevWorkspaceEvents{
 					PostStart: []string{"asdf"},
 					PreStart:  []string{"asdf"},
 				},

pkg/odo/cli/catalog/describe/component.go

@@ -2,7 +2,10 @@ package describe
 
 import (
 	"fmt"
+	"net/url"
 	"os"
+	"path"
+	"strings"
 	"text/tabwriter"
 
 	"github.com/pkg/errors"
@@ -196,11 +199,25 @@ func (o *DescribeComponentOptions) GetDevfileComponentsByName(catalogDevfileList
 // GetDevfile downloads the devfile in memory and return the devfile object
 func GetDevfile(devfileComponent catalog.DevfileComponentType) (parser.DevfileObj, error) {
 	var devObj parser.DevfileObj
+	var err error
 
-	devObj, err := devfile.ParseFromURLAndValidate(devfileComponent.Registry.URL + devfileComponent.Link)
-	if err != nil {
-		return devObj, errors.Wrapf(err, "Failed to download devfile.yaml for devfile component: %s", devfileComponent.Name)
+	if strings.Contains(devfileComponent.Registry.URL, "github") {
+		devObj, err = devfile.ParseFromURLAndValidate(devfileComponent.Registry.URL + devfileComponent.Link)
+		if err != nil {
+			return devObj, errors.Wrapf(err, "Failed to download devfile.yaml from Github-based registry for devfile component: %s", devfileComponent.Name)
+		}
+	} else {
+		registryURL, err := url.Parse(devfileComponent.Registry.URL)
+		if err != nil {
+			return devObj, errors.Wrapf(err, "Failed to parse registry URL for devfile component: %s", devfileComponent.Name)
+		}
+		registryURL.Path = path.Join(registryURL.Path, "devfiles", devfileComponent.Name)
+		devObj, err = devfile.ParseFromURLAndValidate(registryURL.String())
+		if err != nil {
+			return devObj, errors.Wrapf(err, "Failed to download devfile.yaml from OCI-based registry for devfile component: %s", devfileComponent.Name)
+		}
 	}
+
 	err = validate.ValidateDevfileData(devObj.Data)
 	if err != nil {
 		return devObj, err

pkg/odo/cli/component/create.go

@@ -12,6 +12,7 @@ import (
 	"github.com/zalando/go-keyring"
 
 	"github.com/devfile/library/pkg/devfile"
+	registryLibrary "github.com/devfile/registry-support/registry-library/library"
 	"github.com/openshift/odo/pkg/catalog"
 	"github.com/openshift/odo/pkg/component"
 	"github.com/openshift/odo/pkg/config"
@@ -800,23 +801,37 @@ func (co *CreateOptions) devfileRun() (err error) {
 			}
 		} else {
 			// Download devfile from registry
-			params := util.HTTPRequestParams{
-				URL: co.devfileMetadata.devfileRegistry.URL + co.devfileMetadata.devfileLink,
-			}
+			var params util.HTTPRequestParams
 
-			if registryUtil.IsSecure(co.devfileMetadata.devfileRegistry.Name) {
-				token, err := keyring.Get(fmt.Sprintf("%s%s", util.CredentialPrefix, co.devfileMetadata.devfileRegistry.Name), registryUtil.RegistryUser)
+			if strings.Contains(co.devfileMetadata.devfileRegistry.URL, "github") {
+				// Github-based registry
+				params = util.HTTPRequestParams{
+					URL: co.devfileMetadata.devfileRegistry.URL + co.devfileMetadata.devfileLink,
+				}
+				if registryUtil.IsSecure(co.devfileMetadata.devfileRegistry.Name) {
+					token, err := keyring.Get(fmt.Sprintf("%s%s", util.CredentialPrefix, co.devfileMetadata.devfileRegistry.Name), registryUtil.RegistryUser)
+					if err != nil {
+						return errors.Wrap(err, "unable to get secure registry credential from keyring")
+					}
+					params.Token = token
+				}
+			} else {
+				err = registryLibrary.PullStackFromRegistry(co.devfileMetadata.devfileRegistry.URL, co.devfileMetadata.componentType, co.componentContext)
 				if err != nil {
-					return errors.Wrap(err, "unable to get secure registry credential from keyring")
+					return err
 				}
-				params.Token = token
 			}
 
 			cfg, err := preference.New()
 			if err != nil {
 				return err
 			}
-			devfileData, err = util.DownloadFileInMemoryWithCache(params, cfg.GetRegistryCacheTime())
+
+			if strings.Contains(co.devfileMetadata.devfileRegistry.URL, "github") {
+				devfileData, err = util.DownloadFileInMemoryWithCache(params, cfg.GetRegistryCacheTime())
+			} else {
+				devfileData, err = ioutil.ReadFile(DevfilePath)
+			}
 			if err != nil {
 				return errors.Wrapf(err, "failed to download devfile for devfile component from %s", co.devfileMetadata.devfileRegistry.URL+co.devfileMetadata.devfileLink)
 			}

pkg/preference/preference.go

@@ -80,7 +80,7 @@ const (
 	DefaultDevfileRegistryName = "DefaultDevfileRegistry"
 
 	// DefaultDevfileRegistryURL is the URL of default devfile registry
-	DefaultDevfileRegistryURL = "https://github.com/odo-devfiles/registry"
+	DefaultDevfileRegistryURL = "https://registry.devfile.io"
 
 	// DefaultRegistryCacheTime is time (in minutes) for how long odo will cache information from Devfile registry
 	DefaultRegistryCacheTime = 15
@@ -267,6 +267,17 @@ func NewPreferenceInfo() (*PreferenceInfo, error) {
 		c.OdoSettings.RegistryList = &defaultRegistryList
 	}
 
+	// Handle OCI-based default registry migration
+	if c.OdoSettings.RegistryList != nil {
+		for index, registry := range *c.OdoSettings.RegistryList {
+			if registry.Name == DefaultDevfileRegistryName && registry.URL != DefaultDevfileRegistryURL {
+				registryList := *c.OdoSettings.RegistryList
+				registryList[index].URL = DefaultDevfileRegistryURL
+				break
+			}
+		}
+	}
+
 	return &c, nil
 }
 

pkg/util/util.go

@@ -801,25 +801,22 @@ func FilterIgnores(filesChanged, filesDeleted, absIgnoreRules []string) (filesCh
 }
 
 // IsValidProjectDir checks that the folder to download the project from devfile is
-// either empty or only contains the devfile used.
+// either empty or contains the devfile used.
 func IsValidProjectDir(path string, devfilePath string) error {
 	files, err := ioutil.ReadDir(path)
 	if err != nil {
 		return err
 	}
 
-	if len(files) > 1 {
-		return errors.Errorf("Folder %s is not empty. It can only contain the devfile used.", path)
-	} else if len(files) == 1 {
-		file := files[0]
-		if file.IsDir() {
-			return errors.Errorf("Folder %s is not empty. It contains a subfolder.", path)
-		}
-		fileName := files[0].Name()
-		devfilePath = strings.TrimPrefix(devfilePath, "./")
-		if fileName != devfilePath {
-			return errors.Errorf("Folder %s contains one element and it's not the devfile used.", path)
+	if len(files) >= 1 {
+		for _, file := range files {
+			fileName := file.Name()
+			devfilePath = strings.TrimPrefix(devfilePath, "./")
+			if !file.IsDir() && fileName == devfilePath {
+				return nil
+			}
 		}
+		return errors.Errorf("Folder %s doesn't contain the devfile used.", path)
 	}
 
 	return nil

pkg/util/util_test.go

@@ -1571,14 +1571,14 @@ func TestIsValidProjectDir(t *testing.T) {
 			devfilePath:   "devfile.yaml",
 			filesToCreate: []string{"file1.yaml"},
 			dirToCreate:   []string{},
-			expectedError: "Folder %s contains one element and it's not the devfile used.",
+			expectedError: "Folder %s doesn't contain the devfile used.",
 		},
 		{
 			name:          "Case 4: Folder contains a hidden file which is not the devfile",
 			devfilePath:   "devfile.yaml",
 			filesToCreate: []string{".file1.yaml"},
 			dirToCreate:   []string{},
-			expectedError: "Folder %s contains one element and it's not the devfile used.",
+			expectedError: "Folder %s doesn't contain the devfile used.",
 		},
 		{
 			name:          "Case 5: Folder contains devfile.yaml and more files",
@@ -1587,20 +1587,6 @@ func TestIsValidProjectDir(t *testing.T) {
 			dirToCreate:   []string{},
 			expectedError: "Folder %s is not empty. It can only contain the devfile used.",
 		},
-		{
-			name:          "Case 6: Folder contains a directory",
-			devfilePath:   "",
-			filesToCreate: []string{},
-			dirToCreate:   []string{"dir"},
-			expectedError: "Folder %s is not empty. It contains a subfolder.",
-		},
-		{
-			name:          "Case 7: Folder contains a hidden directory",
-			devfilePath:   "",
-			filesToCreate: []string{},
-			dirToCreate:   []string{".dir"},
-			expectedError: "Folder %s is not empty. It contains a subfolder.",
-		},
 	}
 
 	for _, tt := range tests {