Add offline support for starter projects

.ci/Dockerfile

@@ -21,6 +21,9 @@ COPY build-tools /build-tools
 COPY index/ /index
 COPY tests/registry /registry
 
+# Download offline starter projects
+RUN bash /build-tools/dl_starter_projects.sh go-starter community
+
 # Run the registry build tools
 RUN /build-tools/build.sh /registry /build
 

build-tools/build.sh

@@ -30,7 +30,8 @@ tar_files_and_cleanup() {
     -a -not -name "*.vsx" \
     -a -not -name "." \
     -a -not -name "logo.svg" \
-    -a -not -name "logo.png" \) -maxdepth 1)
+    -a -not -name "logo.png" \
+    -a -not -name "*.zip"\) -maxdepth 1)
 
   # There are files that need to be pulled into a tar archive
   if [[ ! -z $tarFiles ]]; then

build-tools/dl_starter_projects.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+if [[ -z "$@" ]]
+then
+    echo "No starter projects specified."
+    exit 0
+fi
+
+# Path of stacks directory in the registry
+STACKS_DIR=/registry/stacks
+# List of starter projects to use offline
+offline_starter_projects=( "$@" )
+
+# Downloads a starter project from a remote git repository and packages it as a zip archive
+# to be used as an offline resource.
+download_git_starter_project() {
+    stack_root=$1
+    name=$2
+    remote_name=$(yq e ".starterProjects[] | select(.name == \"${name}\").git.checkoutFrom.remote" $stack_root/devfile.yaml)
+    revision=$(yq e ".starterProjects[] | select(.name == \"${name}\").git.checkoutFrom.revision" $stack_root/devfile.yaml)
+    subDir=$(yq e ".starterProjects[] | select(.name == \"${name}\").subDir" $stack_root/devfile.yaml)
+    local_path=${stack_root}/${name}-offline
+
+    if [ "${remote_name}" == "null" ]
+    then
+        remote_url=$(yq e ".starterProjects[] | select(.name == \"${name}\").git.remotes.origin" $stack_root/devfile.yaml)
+    else
+        remote_url=$(yq e ".starterProjects[] | select(.name == \"${name}\").git.remotes.${remote_name}" $stack_root/devfile.yaml)
+    fi
+
+    mkdir -p $local_path
+
+    git clone $remote_url $local_path
+
+    if [ "${revision}" != "null" ]
+    then
+        cd $local_path && git checkout $revision && cd -
+    fi
+
+    if [ "${subDir}" != "null" ]
+    then
+        cd $local_path/$subDir && zip -q ${local_path}.zip * .[^.]* && cd -
+    else
+        cd $local_path && rm -rf ./.git && zip -q ${local_path}.zip * .[^.]* && cd -
+    fi
+
+    rm -rf $local_path
+}
+
+# Downloads a starter project from a remote zip archive source
+# to be used as an offline resource.
+download_zip_starter_project() {
+    stack_root=$1
+    name=$2
+    remote_url=$(yq e ".starterProjects[] | select(.name == \"${name}\").zip.location" $stack_root/devfile.yaml)
+    local_path=${stack_root}/${name}-offline
+
+    curl -L $remote_url -o ${local_path}.zip
+}
+
+# Read stacks list
+read -r -a stacks <<< "$(ls ${STACKS_DIR} | tr '\n' ' ')"
+
+echo "Downloading offline starter projects.."
+for starter_project in ${offline_starter_projects[@]}
+do
+    for stack in ${stacks[@]}
+    do
+        stack_root=$STACKS_DIR/$stack
+        stack_devfile=$stack_root/devfile.yaml
+        # Read version list for stack
+        read -r -a versions <<< "$(ls ${STACKS_DIR}/${stack} | grep -e '[0-9].[0-9].[0-9]' | tr '\n' ' ')"
+        # If multi version stack
+        if [[ ${#versions[@]} -gt 0 ]]
+        then
+            for version in ${versions[@]}
+            do
+                stack_root=$STACKS_DIR/$stack/$version
+                stack_devfile=$stack_root/devfile.yaml
+                # If the specified starter project is found
+                if [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ]
+                then
+                    # Starter project has a git remote
+                    if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ]
+                    then
+                        echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}.."
+                        download_git_starter_project $stack_root $starter_project
+                        echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!"
+                    # Starter project has a zip remote
+                    elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ]
+                    then
+                        echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}.."
+                        download_zip_starter_project $stack_root $starter_project
+                        echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!"
+                    fi
+                fi
+            done
+        # If not multi version stack & the specified starter project is found
+        elif [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ]
+        then
+            # Starter project has a git remote
+            if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ]
+            then
+                echo "Downloading ${starter_project} starter project in stack ${stack}.."
+                download_git_starter_project $stack_root $starter_project
+                echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
+            # Starter project has a zip remote
+            elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ]
+            then
+                echo "Downloading ${starter_project} starter project in stack ${stack}.."
+                download_zip_starter_project $stack_root $starter_project
+                echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
+            fi
+        fi
+    done
+done
+echo "Downloading offline starter projects..done!"

index/server/pkg/server/endpoint.go

@@ -15,12 +15,11 @@ import (
 	"github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
 	"github.com/devfile/library/pkg/devfile/parser"
 	"github.com/devfile/library/pkg/devfile/parser/data/v2/common"
+	dfutil "github.com/devfile/library/pkg/util"
 	libutil "github.com/devfile/registry-support/index/generator/library"
-	"github.com/devfile/registry-support/index/generator/schema"
 	indexSchema "github.com/devfile/registry-support/index/generator/schema"
 	"github.com/devfile/registry-support/index/server/pkg/util"
 	"github.com/gin-gonic/gin"
-	versionpkg "github.com/hashicorp/go-version"
 	"github.com/prometheus/client_golang/prometheus"
 	"gopkg.in/segmentio/analytics-go.v3"
 )
@@ -131,7 +130,22 @@ func serveDevfileStarterProjectWithVersion(c *gin.Context) {
 	version := c.Param("version")
 	starterProjectName := c.Param("starterProjectName")
 	downloadTmpLoc := path.Join("/tmp", starterProjectName)
-	devfileBytes, _ := fetchDevfile(c, devfileName, version) // TODO: add devfileIndex when telemetry is migrated
+	stackLoc := path.Join(stacksPath, devfileName)
+	devfileBytes, devfileIndex := fetchDevfile(c, devfileName, version)
+
+	if len(devfileIndex.Versions) > 1 {
+		versionMap, err := util.MakeVersionMap(devfileIndex)
+		if err != nil {
+			log.Print(err.Error())
+			c.JSON(http.StatusInternalServerError, gin.H{
+				"error":  err.Error(),
+				"status": "failed to parse the stack version",
+			})
+			return
+		}
+
+		stackLoc = path.Join(stackLoc, versionMap[version].Version)
+	}
 
 	if len(devfileBytes) == 0 {
 		// fetchDevfile was unsuccessful (error or not found)
@@ -169,7 +183,7 @@ func serveDevfileStarterProjectWithVersion(c *gin.Context) {
 		}
 
 		if starterProject := starterProjects[0]; starterProject.Git != nil {
-			gitScheme := schema.Git{
+			gitScheme := indexSchema.Git{
 				Remotes:    starterProject.Git.Remotes,
 				RemoteName: "origin",
 				SubDir:     starterProject.SubDir,
@@ -194,14 +208,78 @@ func serveDevfileStarterProjectWithVersion(c *gin.Context) {
 				return
 			}
 		} else if starterProject.Zip != nil {
-			downloadBytes, err = libutil.DownloadStackFromZipUrl(starterProject.Zip.Location, starterProject.SubDir, downloadTmpLoc)
-			if err != nil {
-				log.Print(err.Error())
-				c.JSON(http.StatusInternalServerError, gin.H{
-					"error":  err.Error(),
-					"status": fmt.Sprintf("Problem with downloading starter project %s", starterProjectName),
-				})
-				return
+			if _, err = url.ParseRequestURI(starterProject.Zip.Location); err != nil {
+				localLoc := path.Join(stackLoc, starterProject.Zip.Location)
+				log.Printf("zip location is not a valid http url: %v\nTrying local path %s..", err, localLoc)
+
+				// If subdirectory is specified for starter project download then extract subdirectory
+				// and create new archive for download.
+				if starterProject.SubDir != "" {
+					downloadFilePath := fmt.Sprintf("%s.zip", downloadTmpLoc)
+
+					if _, err = os.Stat(downloadTmpLoc); os.IsExist(err) {
+						err = os.Remove(downloadTmpLoc)
+						if err != nil {
+							log.Print(err.Error())
+							c.JSON(http.StatusInternalServerError, gin.H{
+								"error": err.Error(),
+								"status": fmt.Sprintf("Problem removing existing temporary download directory '%s' for starter project %s",
+									downloadTmpLoc,
+									starterProjectName),
+							})
+							return
+						}
+					}
+
+					_, err = dfutil.Unzip(localLoc, downloadTmpLoc, starterProject.SubDir)
+					if err != nil {
+						log.Print(err.Error())
+						c.JSON(http.StatusInternalServerError, gin.H{
+							"error": err.Error(),
+							"status": fmt.Sprintf("Problem with reading subDir '%s' of starter project %s at %s",
+								starterProject.SubDir,
+								starterProjectName,
+								localLoc),
+						})
+						return
+					}
+
+					err = libutil.ZipDir(downloadTmpLoc, downloadFilePath)
+					if err != nil {
+						log.Print(err.Error())
+						c.JSON(http.StatusInternalServerError, gin.H{
+							"error": err.Error(),
+							"status": fmt.Sprintf("Problem with archiving subDir '%s' of starter project %s at %s",
+								starterProject.SubDir,
+								starterProjectName,
+								downloadFilePath),
+						})
+						return
+					}
+
+					localLoc = downloadFilePath
+				}
+
+				downloadBytes, err = ioutil.ReadFile(localLoc)
+				if err != nil {
+					log.Print(err.Error())
+					c.JSON(http.StatusInternalServerError, gin.H{
+						"error": err.Error(),
+						"status": fmt.Sprintf("Problem with reading starter project %s at %s", starterProjectName,
+							localLoc),
+					})
+					return
+				}
+			} else {
+				downloadBytes, err = libutil.DownloadStackFromZipUrl(starterProject.Zip.Location, starterProject.SubDir, downloadTmpLoc)
+				if err != nil {
+					log.Print(err.Error())
+					c.JSON(http.StatusInternalServerError, gin.H{
+						"error":  err.Error(),
+						"status": fmt.Sprintf("Problem with downloading starter project %s", starterProjectName),
+					})
+					return
+				}
 			}
 		} else {
 			c.JSON(http.StatusBadRequest, gin.H{
@@ -430,44 +508,25 @@ func fetchDevfile(c *gin.Context, name string, version string) ([]byte, indexSch
 					sampleDevfilePath = path.Join(samplesPath, devfileIndex.Name, devfileName)
 				}
 			} else {
-				versionMap := make(map[string]indexSchema.Version)
-				var latestVersion string
-				for _, versionElement := range devfileIndex.Versions {
-					versionMap[versionElement.Version] = versionElement
-					if versionElement.Default {
-						versionMap["default"] = versionElement
-					}
-					if latestVersion != "" {
-						latest, err := versionpkg.NewVersion(latestVersion)
-						if err != nil {
-							log.Print(err.Error())
-							c.JSON(http.StatusInternalServerError, gin.H{
-								"error":  err.Error(),
-								"status": fmt.Sprintf("failed to parse the stack version %s for stack %s", latestVersion, name),
-							})
-							return []byte{}, indexSchema.Schema{}
-						}
-						current, err := versionpkg.NewVersion(versionElement.Version)
+				versionMap, err := util.MakeVersionMap(devfileIndex)
+				if err != nil {
+					log.Print(err.Error())
+					c.JSON(http.StatusInternalServerError, gin.H{
+						"error":  err.Error(),
+						"status": "failed to parse the stack version",
+					})
+					return []byte{}, indexSchema.Schema{}
+				}
+				if foundVersion, ok := versionMap[version]; ok {
+					if devfileIndex.Type == indexSchema.StackDevfileType {
+						bytes, err = pullStackFromRegistry(foundVersion)
 						if err != nil {
-							log.Print(err.Error())
 							c.JSON(http.StatusInternalServerError, gin.H{
 								"error":  err.Error(),
-								"status": fmt.Sprintf("failed to parse the stack version %s for stack %s", versionElement.Version, name),
+								"status": fmt.Sprintf("Problem pulling version %s from OCI Registry", foundVersion.Version),
 							})
 							return []byte{}, indexSchema.Schema{}
 						}
-						if current.GreaterThan(latest) {
-							latestVersion = versionElement.Version
-						}
-					} else {
-						latestVersion = versionElement.Version
-					}
-				}
-				versionMap["latest"] = versionMap[latestVersion]
-
-				if foundVersion, ok := versionMap[version]; ok {
-					if devfileIndex.Type == indexSchema.StackDevfileType {
-						bytes, err = pullStackFromRegistry(foundVersion)
 					} else {
 						// Retrieve the sample devfile stored under /registry/samples/<devfile>
 						sampleDevfilePath = path.Join(samplesPath, devfileIndex.Name, foundVersion.Version, devfileName)

index/server/pkg/server/registry.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 
 	indexSchema "github.com/devfile/registry-support/index/generator/schema"
 
@@ -24,8 +25,8 @@ func pushStackToRegistry(versionComponent indexSchema.Version, stackName string)
 	memoryStore := content.NewMemoryStore()
 	pushContents := []ocispec.Descriptor{}
 	for _, resource := range versionComponent.Resources {
-		if resource == "meta.yaml" {
-			// Some registries may still have the meta.yaml in it, but we don't need it, so skip pushing it up
+		if resource == "meta.yaml" || strings.HasSuffix(resource, "-offline.zip") {
+			// Some registries may still have the meta.yaml (we don't need it) or offline resources in it, so skip pushing these up
 			continue
 		}
 

index/server/pkg/util/util.go

@@ -11,6 +11,8 @@ import (
 	"strconv"
 	"strings"
 
+	versionpkg "github.com/hashicorp/go-version"
+
 	indexLibrary "github.com/devfile/registry-support/index/generator/library"
 	indexSchema "github.com/devfile/registry-support/index/generator/schema"
 )
@@ -182,3 +184,32 @@ func IsTelemetryEnabled() bool {
 	}
 	return false
 }
+
+// MakeVersionMap creates a map of versions for a given devfile index schema.
+func MakeVersionMap(devfileIndex indexSchema.Schema) (map[string]indexSchema.Version, error) {
+	versionMap := make(map[string]indexSchema.Version)
+	var latestVersion string
+	for _, versionElement := range devfileIndex.Versions {
+		versionMap[versionElement.Version] = versionElement
+		if versionElement.Default {
+			versionMap["default"] = versionElement
+		}
+		if latestVersion != "" {
+			latest, err := versionpkg.NewVersion(latestVersion)
+			if err != nil {
+				return map[string]indexSchema.Version{}, err
+			}
+			current, err := versionpkg.NewVersion(versionElement.Version)
+			if err != nil {
+				return map[string]indexSchema.Version{}, err
+			}
+			if current.GreaterThan(latest) {
+				latestVersion = versionElement.Version
+			}
+		} else {
+			latestVersion = versionElement.Version
+		}
+	}
+	versionMap["latest"] = versionMap[latestVersion]
+	return versionMap, nil
+}