Card origin_devexp_286 - Added ssh key-based access to private git repository

assets/app/scripts/controllers/create.js

@@ -14,7 +14,7 @@ angular.module('openshiftConsole')
 
     $scope.templatesByTag = {};
 
-    $scope.sourceURLPattern = /^(ftp|http|https|git):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?$/;
+    $scope.sourceURLPattern = /^((ftp|http|https|git):\/\/(\w+:{0,1}\w*@)|git@)?([^\s@]+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?$/;
 
     DataService.list("templates", $scope, function(templates) {
       $scope.projectTemplates = templates.by("metadata.name");

assets/test/spec/controllers/create.js

@@ -0,0 +1,70 @@
+"use strict";
+
+describe("CreateController", function(){
+  var controller, form;
+  var $scope = {
+    projectTemplates: {},
+    openshiftTemplates: {},
+    templatesByTag: {}
+  };
+
+  beforeEach(function(){
+    inject(function(_$controller_){
+      // The injector unwraps the underscores (_) from around the parameter names when matching
+      controller = _$controller_("CreateController", {
+        $scope: $scope,
+        DataService: {
+          list: function(templates){}
+        }
+      });
+    });
+  });
+
+
+  it("valid http URL", function(){
+    var match = 'http://example.com/dir1/dir2'.match($scope.sourceURLPattern)
+    expect(match).not.toBeNull();
+  });
+
+  it("valid http URL, without http part", function(){
+    var match = 'example.com/dir1/dir2'.match($scope.sourceURLPattern)
+    expect(match).not.toBeNull();
+  });
+
+
+  it("valid http URL with user and password", function(){
+    var match = 'http://user:[email protected]/dir1/dir2'.match($scope.sourceURLPattern)
+    expect(match).not.toBeNull();
+  });
+
+  it("valid http URL with port", function(){
+    var match = 'http://example.com:8080/dir1/dir2'.match($scope.sourceURLPattern)
+    expect(match).not.toBeNull();
+  });
+
+  it("valid http URL with port, user and password", function(){
+    var match = 'http://user:[email protected]:8080/dir1/dir2'.match($scope.sourceURLPattern)
+    expect(match).not.toBeNull();
+  });
+
+  it("valid https URL", function(){
+    var match = 'https://example.com/dir1/dir2'.match($scope.sourceURLPattern)
+    expect(match).not.toBeNull();
+  });
+
+  it("valid ftp URL", function(){
+    var match = 'ftp://example.com/dir1/dir2'.match($scope.sourceURLPattern)
+    expect(match).not.toBeNull();
+  });
+
+  it("valid git+ssh URL", function(){
+    var match = '[email protected]:dir1/dir2'.match($scope.sourceURLPattern)
+    expect(match).not.toBeNull();
+  });
+
+  it("invalid git+ssh URL (double @@)", function(){
+    var match = 'git@@example.com:dir1/dir2'.match($scope.sourceURLPattern)
+    expect(match).toBeNull();
+  });
+
+});

pkg/assets/bindata.go

@@ -16383,7 +16383,7 @@ namespace:"openshift"
 }), g.info("openshift image repos", a.openshiftImageRepos);
 });
 } ]), angular.module("openshiftConsole").controller("CreateController", [ "$scope", "DataService", "$filter", "LabelFilter", "$location", "Logger", function(a, b, c, d, e, f) {
-a.projectTemplates = {}, a.openshiftTemplates = {}, a.templatesByTag = {}, a.sourceURLPattern = /^(ftp|http|https|git):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?$/, b.list("templates", a, function(b) {
+a.projectTemplates = {}, a.openshiftTemplates = {}, a.templatesByTag = {}, a.sourceURLPattern = /^((ftp|http|https|git):\/\/(\w+:{0,1}\w*@)|git@)?([^\s@]+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?$/, b.list("templates", a, function(b) {
 a.projectTemplates = b.by("metadata.name"), g(), f.info("project templates", a.projectTemplates);
 }), b.list("templates", {
 namespace:"openshift"

pkg/build/api/types.go

@@ -111,6 +111,13 @@ type BuildSource struct {
 	// This allows to have buildable sources in directory other than root of
 	// repository.
 	ContextDir string `json:"contextDir,omitempty"`
+
+	// SourceSecretName is the name of a Secret that would be used for setting
+	// up the authentication for cloning private repository.
+	// The secret contains valid credentials for remote repository, where the
+	// data's key represent the authentication method to be used and value is
+	// the base64 encoded credentials. Supported auth methods are: ssh-privatekey.
+	SourceSecretName string
 }
 
 // SourceRevision is the revision or commit information from the source for the build
@@ -240,7 +247,7 @@ type BuildOutput struct {
 	// a Docker image repository to push to. Failure to find the To will result in a build error.
 	To *kapi.ObjectReference `json:"to,omitempty"`
 
-	// pushSecretName is the name of a Secret that would be used for setting
+	// PushSecretName is the name of a Secret that would be used for setting
 	// up the authentication for executing the Docker push to authentication
 	// enabled Docker Registry (or Docker Hub).
 	PushSecretName string `json:"pushSecretName,omitempty"`

pkg/build/api/v1beta1/types.go

@@ -111,6 +111,13 @@ type BuildSource struct {
 	// This allows to have buildable sources in directory other than root of
 	// repository.
 	ContextDir string `json:"contextDir,omitempty"`
+
+	// SourceSecretName is the name of a Secret that would be used for setting
+	// up the authentication for cloning private repository.
+	// The secret contains valid credentials for remote repository, where the
+	// secret's data key represent the authentication method to be used and value is
+	// the base64 encoded credentials. Supported auth methods are: ssh-privatekey.
+	SourceSecretName string `json:"sourceSecretName,omitempty" description:"supported auth methods are: ssh-privatekey`
 }
 
 // SourceRevision is the revision or commit information from the source for the build
@@ -278,7 +285,7 @@ type BuildOutput struct {
 	// a Docker image repository to push to.
 	To *kapi.ObjectReference `json:"to,omitempty"`
 
-	// pushSecretName is the name of a Secret that would be used for setting
+	// PushSecretName is the name of a Secret that would be used for setting
 	// up the authentication for executing the Docker push to authentication
 	// enabled Docker Registry (or Docker Hub).
 	PushSecretName string `json:"pushSecretName,omitempty"`

pkg/build/api/v1beta3/types.go

@@ -117,6 +117,13 @@ type BuildSource struct {
 	// This allows to have buildable sources in directory other than root of
 	// repository.
 	ContextDir string `json:"contextDir,omitempty"`
+
+	// SourceSecretName is the name of a Secret that would be used for setting
+	// up the authentication for cloning private repository.
+	// The secret contains valid credentials for remote repository, where the
+	// data's key represent the authentication method to be used and value is
+	// the base64 encoded credentials. Supported auth methods are: ssh-privatekey.
+	SourceSecretName string `json:"sourceSecretName,omitempty" description:"supported auth methods are: ssh-privatekey`
 }
 
 // SourceRevision is the revision or commit information from the source for the build

pkg/build/builder/cmd/builder.go

@@ -1,14 +1,19 @@
 package cmd
 
 import (
+	"fmt"
+	"io/ioutil"
 	"os"
+	"os/exec"
+	"path/filepath"
 
 	"github.com/fsouza/go-dockerclient"
 	"github.com/golang/glog"
 	"github.com/openshift/origin/pkg/api/latest"
 	"github.com/openshift/origin/pkg/build/api"
 	bld "github.com/openshift/origin/pkg/build/builder"
 	"github.com/openshift/origin/pkg/build/builder/cmd/dockercfg"
+	"github.com/openshift/origin/pkg/build/builder/cmd/scmauth"
 	dockerutil "github.com/openshift/origin/pkg/cmd/util/docker"
 	image "github.com/openshift/origin/pkg/image/api"
 )
@@ -19,14 +24,17 @@ const DockerCfgFile = ".dockercfg"
 type builder interface {
 	Build() error
 }
+
 type factoryFunc func(
 	client bld.DockerClient,
 	dockerSocket string,
 	authConfig docker.AuthConfiguration,
 	authPresent bool,
 	build *api.Build) builder
 
-func run(builderFactory factoryFunc) {
+// run is responsible for preparing environment for actual build.
+// It accepts factoryFunc and an ordered array of SCMAuths.
+func run(builderFactory factoryFunc, scmAuths []scmauth.SCMAuth) {
 	client, endpoint, err := dockerutil.NewHelper().GetClient()
 	if err != nil {
 		glog.Fatalf("Error obtaining docker client: %v", err)
@@ -57,6 +65,11 @@ func run(builderFactory factoryFunc) {
 			dockercfg.PullAuthType,
 		)
 	}
+	if len(build.Parameters.Source.SourceSecretName) > 0 {
+		if err := setupSourceSecret(build.Parameters.Source.SourceSecretName, scmAuths); err != nil {
+			glog.Fatalf("Cannot setup secret file for accessing private repository: %v", err)
+		}
+	}
 	b := builderFactory(client, endpoint, authcfg, authPresent, &build)
 	if err = b.Build(); err != nil {
 		glog.Fatalf("Build error: %v", err)
@@ -67,16 +80,83 @@ func run(builderFactory factoryFunc) {
 
 }
 
+// fixSecretPermissions loweres access permissions to very low acceptable level
+// TODO: this method should be removed as soon as secrets permissions are fixed upstream
+func fixSecretPermissions() error {
+	secretTmpDir, err := ioutil.TempDir("", "tmpsecret")
+	if err != nil {
+		return err
+	}
+	cmd := exec.Command("cp", "-R", ".", secretTmpDir)
+	cmd.Dir = os.Getenv("SOURCE_SECRET_PATH")
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+	secretFiles, err := ioutil.ReadDir(secretTmpDir)
+	if err != nil {
+		return err
+	}
+	for _, file := range secretFiles {
+		if err := os.Chmod(filepath.Join(secretTmpDir, file.Name()), 0600); err != nil {
+			return err
+		}
+	}
+	os.Setenv("SOURCE_SECRET_PATH", secretTmpDir)
+	return nil
+}
+
+func setupSourceSecret(sourceSecretName string, scmAuths []scmauth.SCMAuth) error {
+	fixSecretPermissions()
+	sourceSecretDir := os.Getenv("SOURCE_SECRET_PATH")
+	files, err := ioutil.ReadDir(sourceSecretDir)
+	if err != nil {
+		return err
+	}
+	found := false
+
+SCMAuthLoop:
+	for _, scmAuth := range scmAuths {
+		glog.V(3).Infof("Checking for '%s' in secret '%s'", scmAuth.Name(), sourceSecretName)
+		for _, file := range files {
+			if file.Name() == scmAuth.Name() {
+				glog.Infof("Using '%s' from secret '%s'", scmAuth.Name(), sourceSecretName)
+				if err := scmAuth.Setup(sourceSecretDir); err != nil {
+					glog.Warningf("Error setting up '%s': %v", scmAuth.Name(), err)
+					continue
+				}
+				found = true
+				break SCMAuthLoop
+			}
+		}
+	}
+	if !found {
+		return fmt.Errorf("the provided secret '%s' did not have any of the supported keys %v",
+			sourceSecretName, getSCMNames(scmAuths))
+	}
+	return nil
+}
+
+func getSCMNames(scmAuths []scmauth.SCMAuth) string {
+	var names string
+	for _, scmAuth := range scmAuths {
+		if len(names) > 0 {
+			names += ", "
+		}
+		names += scmAuth.Name()
+	}
+	return names
+}
+
 // RunDockerBuild creates a docker builder and runs its build
 func RunDockerBuild() {
 	run(func(client bld.DockerClient, sock string, auth docker.AuthConfiguration, present bool, build *api.Build) builder {
 		return bld.NewDockerBuilder(client, auth, present, build)
-	})
+	}, []scmauth.SCMAuth{&scmauth.SSHPrivateKey{}})
 }
 
 // RunSTIBuild creates a STI builder and runs its build
 func RunSTIBuild() {
 	run(func(client bld.DockerClient, sock string, auth docker.AuthConfiguration, present bool, build *api.Build) builder {
 		return bld.NewSTIBuilder(client, sock, auth, present, build)
-	})
+	}, []scmauth.SCMAuth{&scmauth.SSHPrivateKey{}})
 }

pkg/build/builder/cmd/dockercfg/cfg.go

@@ -47,7 +47,7 @@ func (h *Helper) GetDockerAuth(registry, authType string) (docker.AuthConfigurat
 		dockercfgPath = getDockercfgFile(pathForAuthType)
 	}
 	if _, err := os.Stat(dockercfgPath); err != nil {
-		glog.V(3).Infof("%s: %v", dockercfgPath, err)
+		glog.V(3).Infof("Problem accessing %s: %v", dockercfgPath, err)
 		return authCfg, false
 	}
 	cfg, err := readDockercfg(dockercfgPath)

pkg/build/builder/cmd/scmauth/scmauth.go

@@ -0,0 +1,9 @@
+package scmauth
+
+// SCMAuth is an interface implemented by different authentication providers
+// which are responsible for setting up the credentials to be used when accessing
+// private repository.
+type SCMAuth interface {
+	Name() string
+	Setup(baseDir string) error
+}

pkg/build/builder/cmd/scmauth/sshkey.go

@@ -0,0 +1,40 @@
+package scmauth
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+const SSHPrivateKeyMethodName = "ssh-privatekey"
+
+// SSHPrivateKey implements SCMAuth interface for using SSH private keys.
+type SSHPrivateKey struct{}
+
+// Setup creates a wrapper script for SSH command to be able to use the provided
+// SSH key while accessing private repository.
+func (_ SSHPrivateKey) Setup(baseDir string) error {
+	script, err := ioutil.TempFile("", "gitssh")
+	if err != nil {
+		return err
+	}
+	defer script.Close()
+	if err := script.Chmod(0711); err != nil {
+		return err
+	}
+	if _, err := script.WriteString("#!/bin/sh\nssh -i " +
+		filepath.Join(baseDir, SSHPrivateKeyMethodName) +
+		" -o StrictHostKeyChecking=false \"$@\"\n"); err != nil {
+		return err
+	}
+	// set environment variable to tell git to use the SSH wrapper
+	if err := os.Setenv("GIT_SSH", script.Name()); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Name returns the name of this auth method.
+func (_ SSHPrivateKey) Name() string {
+	return SSHPrivateKeyMethodName
+}

pkg/build/controller/strategy/custom.go

@@ -79,5 +79,6 @@ func (bs *CustomBuildStrategy) CreateBuildPod(build *buildapi.Build) (*kapi.Pod,
 		setupDockerSocket(pod)
 		setupDockerSecrets(pod, build.Parameters.Output.PushSecretName)
 	}
+	setupSourceSecrets(pod, build.Parameters.Source.SourceSecretName)
 	return pod, nil
 }

pkg/build/controller/strategy/custom_test.go

@@ -53,21 +53,19 @@ func TestCustomCreateBuildPod(t *testing.T) {
 	if actual.Spec.RestartPolicy != kapi.RestartPolicyNever {
 		t.Errorf("Expected never, got %#v", actual.Spec.RestartPolicy)
 	}
-	if len(container.VolumeMounts) != 2 {
-		t.Fatalf("Expected 2 volumes in container, got %d", len(container.VolumeMounts))
+	if len(container.VolumeMounts) != 3 {
+		t.Fatalf("Expected 3 volumes in container, got %d", len(container.VolumeMounts))
 	}
-	if container.VolumeMounts[0].MountPath != dockerSocketPath {
-		t.Fatalf("Expected %s in first VolumeMount, got %s", dockerSocketPath, container.VolumeMounts[0].MountPath)
-	}
-	if container.VolumeMounts[1].MountPath != dockerPushSecretMountPath {
-		t.Fatalf("Expected %s in first VolumeMount, got %s", dockerPushSecretMountPath, container.VolumeMounts[1].MountPath)
+	for i, expected := range []string{dockerSocketPath, dockerPushSecretMountPath, sourceSecretMountPath} {
+		if container.VolumeMounts[i].MountPath != expected {
+			t.Fatalf("Expected %s in VolumeMount[%d], got %s", expected, i, container.VolumeMounts[i].MountPath)
+		}
 	}
 	if !kapi.Semantic.DeepEqual(container.Resources, expected.Parameters.Resources) {
 		t.Fatalf("Expected actual=expected, %v != %v", container.Resources, expected.Parameters.Resources)
 	}
-
-	if len(actual.Spec.Volumes) != 2 {
-		t.Fatalf("Expected 2 volumes in Build pod, got %d", len(actual.Spec.Volumes))
+	if len(actual.Spec.Volumes) != 3 {
+		t.Fatalf("Expected 3 volumes in Build pod, got %d", len(actual.Spec.Volumes))
 	}
 	buildJSON, _ := v1beta1.Codec.Encode(expected)
 	errorCases := map[int][]string{
@@ -110,6 +108,7 @@ func mockCustomBuild() *buildapi.Build {
 					URI: "http://my.build.com/the/dockerbuild/Dockerfile",
 					Ref: "master",
 				},
+				SourceSecretName: "secretFoo",
 			},
 			Strategy: buildapi.BuildStrategy{
 				Type: buildapi.CustomBuildStrategyType,

pkg/build/controller/strategy/docker.go

@@ -55,5 +55,6 @@ func (bs *DockerBuildStrategy) CreateBuildPod(build *buildapi.Build) (*kapi.Pod,
 
 	setupDockerSocket(pod)
 	setupDockerSecrets(pod, build.Parameters.Output.PushSecretName)
+	setupSourceSecrets(pod, build.Parameters.Source.SourceSecretName)
 	return pod, nil
 }