Feature/contextsources (#195)

* added switch to extract different sources as build context

* first rough implementation of aws s3

* added buildcontext package and interface

* added GetBuildContext func to buildcontext.go
added fallback to gcs
renamed GC struct to GCS

* improved the default behavior of build context retrieval

* renamed gc:// to gs:// in order to follow common standards

* renamed struct File to Dir and some cleanup work

* moved context.tar suffix to the buildcontext processors where it is needed

* added buildcontext retrieval as struct variable

added fallback if prefix in bucket specifier is present

* cleanup if structures

* added prefix to s3

* WIP

* Fixed build context bugs

* refactored build context

Gopkg.toml

@@ -46,3 +46,7 @@
 [[constraint]]
   name = "github.com/genuinetools/amicontained"
   version = "0.4.0"
+
+[[constraint]]
+  name = "github.com/aws/aws-sdk-go"
+  version = "1.13.56"

cmd/executor/cmd/root.go

@@ -20,7 +20,9 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
+	"strings"
 
+	"github.com/GoogleContainerTools/kaniko/pkg/buildcontext"
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"github.com/GoogleContainerTools/kaniko/pkg/executor"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
@@ -121,21 +123,32 @@ func checkDockerfilePath() error {
 	return errors.New("please provide a valid path to a Dockerfile within the build context")
 }
 
-// resolveSourceContext unpacks the source context if it is a tar in a GCS bucket
+// resolveSourceContext unpacks the source context if it is a tar in a bucket
 // it resets srcContext to be the path to the unpacked build context within the image
 func resolveSourceContext() error {
 	if srcContext == "" && bucket == "" {
-		return errors.New("please specify a path to the build context with the --context flag or a GCS bucket with the --bucket flag")
+		return errors.New("please specify a path to the build context with the --context flag or a bucket with the --bucket flag")
 	}
-	if bucket == "" {
+	if srcContext != "" && !strings.Contains(srcContext, "://") {
 		return nil
 	}
-	logrus.Infof("Using GCS bucket %s as source context", bucket)
-	buildContextPath := constants.BuildContextDir
-	if err := util.UnpackTarFromGCSBucket(bucket, buildContextPath); err != nil {
+	if bucket != "" {
+		if !strings.Contains(bucket, "://") {
+			srcContext = constants.GCSBuildContextPrefix + bucket
+		} else {
+			srcContext = bucket
+		}
+	}
+	// if no prefix use Google Cloud Storage as default for backwards compability
+	contextExecutor, err := buildcontext.GetBuildContext(srcContext)
+	if err != nil {
+		return err
+	}
+	logrus.Debugf("Getting source context from %s", srcContext)
+	srcContext, err = contextExecutor.UnpackTarFromBuildContext()
+	if err != nil {
 		return err
 	}
-	logrus.Debugf("Unpacked tar from %s to path %s", bucket, buildContextPath)
-	srcContext = buildContextPath
+	logrus.Debugf("Build context located at %s", srcContext)
 	return nil
 }

integration/integration_test.go

@@ -19,6 +19,9 @@ package integration
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/daemon"
 	"math"
 	"os"
 	"os/exec"
@@ -28,9 +31,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/google/go-containerregistry/pkg/name"
-	"github.com/google/go-containerregistry/pkg/v1/daemon"
-
 	"github.com/GoogleContainerTools/kaniko/testutil"
 )
 
@@ -159,7 +159,7 @@ func TestRun(t *testing.T) {
 			for _, d := range bucketContextTests {
 				if d == dockerfile {
 					contextFlag = "-b"
-					contextPath = kanikoTestBucket
+					contextPath = constants.GCSBuildContextPrefix + kanikoTestBucket
 					break
 				}
 			}

pkg/buildcontext/buildcontext.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildcontext
+
+import (
+	"errors"
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"strings"
+)
+
+// BuildContext unifies calls to download and unpack the build context.
+type BuildContext interface {
+	// Unpacks a build context and returns the directory where it resides
+	UnpackTarFromBuildContext() (string, error)
+}
+
+// GetBuildContext parses srcContext for the prefix and returns related buildcontext
+// parser
+func GetBuildContext(srcContext string) (BuildContext, error) {
+	split := strings.SplitAfter(srcContext, "://")
+	prefix := split[0]
+	context := split[1]
+	switch prefix {
+	case constants.GCSBuildContextPrefix:
+		return &GCS{context: context}, nil
+	case constants.S3BuildContextPrefix:
+		return &S3{context: context}, nil
+	case constants.LocalDirBuildContextPrefix:
+		return &Dir{context: context}, nil
+	}
+	return nil, errors.New("unknown build context prefix provided, please use one of the following: gs://, dir://, s3://")
+}

pkg/buildcontext/dir.go

@@ -0,0 +1,27 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildcontext
+
+// Dir unifies calls to download and unpack the build context.
+type Dir struct {
+	context string
+}
+
+// UnpackTarFromBuildContext just provides a directory with already extracted content
+func (f *Dir) UnpackTarFromBuildContext() (string, error) {
+	return f.context, nil
+}

pkg/buildcontext/gcs.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildcontext
+
+import (
+	"os"
+	"path/filepath"
+
+	"cloud.google.com/go/storage"
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+)
+
+// GCS struct for Google Cloud Storage processing
+type GCS struct {
+	context string
+}
+
+func (g *GCS) UnpackTarFromBuildContext() (string, error) {
+	bucket, item := util.GetBucketAndItem(g.context)
+	return constants.BuildContextDir, unpackTarFromGCSBucket(bucket, item, constants.BuildContextDir)
+}
+
+// unpackTarFromGCSBucket unpacks the context.tar.gz file in the given bucket to the given directory
+func unpackTarFromGCSBucket(bucketName, item, directory string) error {
+	// Get the tar from the bucket
+	tarPath, err := getTarFromBucket(bucketName, item, directory)
+	if err != nil {
+		return err
+	}
+	logrus.Debug("Unpacking source context tar...")
+	if err := util.UnpackCompressedTar(tarPath, directory); err != nil {
+		return err
+	}
+	// Remove the tar so it doesn't interfere with subsequent commands
+	logrus.Debugf("Deleting %s", tarPath)
+	return os.Remove(tarPath)
+}
+
+// getTarFromBucket gets context.tar.gz from the GCS bucket and saves it to the filesystem
+// It returns the path to the tar file
+func getTarFromBucket(bucketName, item, directory string) (string, error) {
+	ctx := context.Background()
+	client, err := storage.NewClient(ctx)
+	if err != nil {
+		return "", err
+	}
+	bucket := client.Bucket(bucketName)
+	// Get the tarfile context.tar.gz from the GCS bucket, and save it to a tar object
+	reader, err := bucket.Object(item).NewReader(ctx)
+	if err != nil {
+		return "", err
+	}
+	defer reader.Close()
+	tarPath := filepath.Join(directory, constants.ContextTar)
+	if err := util.CreateFile(tarPath, reader, 0600, 0, 0); err != nil {
+		return "", err
+	}
+	logrus.Debugf("Copied tarball %s from GCS bucket %s to %s", constants.ContextTar, bucketName, tarPath)
+	return tarPath, nil
+}

pkg/buildcontext/s3.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildcontext
+
+import (
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"os"
+	"path/filepath"
+)
+
+// S3 unifies calls to download and unpack the build context.
+type S3 struct {
+	context string
+}
+
+// UnpackTarFromBuildContext download and untar a file from s3
+func (s *S3) UnpackTarFromBuildContext() (string, error) {
+	bucket, item := util.GetBucketAndItem(s.context)
+	downloader := s3manager.NewDownloader(session.New())
+	directory := constants.BuildContextDir
+	tarPath := filepath.Join(directory, constants.ContextTar)
+	if err := os.MkdirAll(directory, 0750); err != nil {
+		return directory, err
+	}
+	file, err := os.Create(tarPath)
+	if err != nil {
+		return directory, err
+	}
+	_, err = downloader.Download(file,
+		&s3.GetObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String(item),
+		})
+	if err != nil {
+		return directory, err
+	}
+
+	return directory, util.UnpackCompressedTar(tarPath, directory)
+}

pkg/constants/constants.go

@@ -46,6 +46,10 @@ const (
 
 	// NoBaseImage is the scratch image
 	NoBaseImage = "scratch"
+
+	GCSBuildContextPrefix      = "gs://"
+	S3BuildContextPrefix       = "s3://"
+	LocalDirBuildContextPrefix = "dir://"
 )
 
 // KanikoBuildFiles is the list of files required to build kaniko