intel · Ajay191191 · Jul 13, 2018 · Sep 13, 2018
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -35,12 +35,20 @@ jobs:
     steps:
       - checkout
       - run:
-          name: install chart
+          name: install gcloud and setup credentials
           command: |
-            mkdir -p $HOME/.kube
-            KUBE_CONFIG=${GKE_KUBECONFIG_KVCCI}
-            echo "${KUBE_CONFIG}" | base64 --decode > "${HOME}/.kube/config"
+            echo ${GKE_ARGO_CREDS} | base64 --decode > ${HOME}/gcloud-service-key.json
+            gcloud auth activate-service-account --key-file ${HOME}/gcloud-service-key.json
+            gcloud --project ${GKE_PROJECT} container clusters create "vck-${CIRCLE_BUILD_NUM}" --zone "${GKE_ZONE}" --cluster-version "${GKE_CLUSTER_VERSION}" --num-nodes "${GKE_NUM_NODES}" --machine-type "${GKE_MACHINE_TYPE}"
             kubectl create ns "vck-${CIRCLE_BUILD_NUM}"
+            # Bootstrap the cluster
+            sh /go/src/github.com/IntelAI/vck/resources/scripts/bootstrap.sh vck-${CIRCLE_BUILD_NUM} ${MINIO_ACCESS_KEY_ID} ${MINIO_SECRET_ACCESS_KEY} true
+      - run:
+          name: install chart
+          command: |
+            export S3_SERVER_IP=minio-vck-${CIRCLE_BUILD_NUM}.vck-${CIRCLE_BUILD_NUM}.svc
+            export NFS_SERVER_IP=nfs-vck-${CIRCLE_BUILD_NUM}-nfs-server-provisioner.vck-${CIRCLE_BUILD_NUM}.svc
+            export PACHYDERM_ADDRESS=pachd.vck-${CIRCLE_BUILD_NUM}.svc:650
             kubectl create secret generic s3-creds \
               --from-literal=awsAccessKeyID=${MINIO_ACCESS_KEY_ID} \
               --from-literal=awsSecretAccessKey=${MINIO_SECRET_ACCESS_KEY} \
@@ -53,16 +61,18 @@ jobs:
               --set crd.install=true \
               --set tag=$(git describe --tags --always --dirty) \
               --set log_level=4
-            export S3_SERVER_IP=$(kubectl get svc ${S3_SVC_NAME} -o json | jq -r .spec.clusterIP)
-            export NFS_SERVER_IP=$(kubectl get svc ${NFS_SERVER_SVC_NAME} -o json | jq -r .spec.clusterIP)
             make dep-ensure
             make code-generation
-            go test -v ./test/e2e/... -args -namespace=vck-${CIRCLE_BUILD_NUM} -s3serverip=${S3_SERVER_IP} -nfsserverip=${NFS_SERVER_IP}
+            go test -v ./test/e2e/... -args -namespace=vck-${CIRCLE_BUILD_NUM} -s3serverip=${S3_SERVER_IP} -nfsserverip=${NFS_SERVER_IP} -pachydermaddress=${PACHYDERM_ADDRESS}
       - run:
           name: cleanup
           command: |
             helm delete --purge vck-${CIRCLE_BUILD_NUM} || true
+            helm delete --purge minio-vck-${CIRCLE_BUILD_NUM} || true
+            helm delete --purge nfs-vck-${CIRCLE_BUILD_NUM} || true
+            helm delete --purge pachyderm-vck-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns vck-${CIRCLE_BUILD_NUM} || true
+            gcloud container clusters delete "vck-${CIRCLE_BUILD_NUM}" --zone "us-west1-a" --project "${GKE_PROJECT}" --quiet
           when: always
 
 workflows:

diff --git a/helm-charts/bootstrap/Chart.yaml b/helm-charts/bootstrap/Chart.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+description: kube-volume-controller-e2e-bootstrap
+name: kube-volume-controller-e2e-bootstrap
+version: v0.1.0
diff --git a/helm-charts/bootstrap/templates/minio_pod.yaml b/helm-charts/bootstrap/templates/minio_pod.yaml
@@ -0,0 +1,18 @@
+{{- if .Values.minio.bootstrap -}}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: minio-bootstrap
+  namespace: {{ .Values.namespace }}
+spec:
+  restartPolicy: OnFailure
+  containers:
+  - command: ["/bin/sh"]
+    args:
+    - -c
+    - mc config host add s3 {{ .Values.minio.server_address }} {{ .Values.minio.access_key }} {{ .Values.minio.secret_key }} &&
+      wget http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz &&
+      mc mb s3/e2e-test && mc cp cifar-100-python.tar.gz s3/e2e-test/cifar-100-python.tar.gz
+    image: minio/mc:RELEASE.2018-02-09T23-07-36Z
+    name: minio
+{{end}}
diff --git a/helm-charts/bootstrap/templates/pachyderm_pod.yaml b/helm-charts/bootstrap/templates/pachyderm_pod.yaml
@@ -0,0 +1,19 @@
+{{- if .Values.pachyderm.bootstrap -}}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pachyderm-bootstrap
+  namespace: {{ .Values.namespace }}
+spec:
+  restartPolicy: OnFailure
+  containers:
+  - command:
+    - bash
+    - -c
+    - export ADDRESS={{ .Values.pachyderm.address }} && pachctl version &&
+      pachctl create-repo test && pachctl create-branch test master &&
+      pachctl start-commit test master && touch /test &&
+      pachctl put-file test master s3/test -f /test && pachctl finish-commit test master
+    image: volumecontroller/pachctl
+    name: pachyderm
+{{end}}
diff --git a/helm-charts/bootstrap/values.yaml b/helm-charts/bootstrap/values.yaml
@@ -0,0 +1,12 @@
+
+namespace:
+
+pachyderm:
+  bootstrap: false
+  address: "pachyderm.default.svc:650"
+
+minio:
+  bootstrap: false
+  server_address: ""
+  access_key: ""
+  secret_key: ""
diff --git a/resources/scripts/bootstrap.sh b/resources/scripts/bootstrap.sh
@@ -0,0 +1,79 @@
+#!/bin/sh
+
+### PS: Assumes a namespace with name $1 exists
+
+set -e
+
+NAMESPACE=$1
+NFS_NAME=nfs-${NAMESPACE}
+PACHYDERM_NAME=pachyderm-${NAMESPACE}
+MINIO_NAME=minio-${NAMESPACE}
+
+MINIO_ACCESS_KEY_ID=$2
+MINIO_SECRET_ACCESS_KEY=$3
+INSTALL_HELM=$4
+
+tryuntil() {
+    COMMAND=$1
+    TARGET=$2
+    TRIES=${3:-100}
+    until eval $1 || [ $TRIES -eq 0 ]; do
+        sleep 10
+        echo "Waiting for ${TARGET}. Tries: $TRIES/${3:-100}"
+        TRIES=$(( TRIES-1 ))
+    done
+
+    if [ $TRIES -eq 0 ]; then
+        exit 1
+    fi
+}
+
+if [ "x$INSTALL_HELM" = "xtrue" ]; then
+    # Install the service account and the cluster role binding
+    echo "Initializing helm.."
+    kubectl create serviceaccount tiller --namespace=kube-system
+cat <<EOF | kubectl create -f -
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: tiller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: tiller
+    namespace: kube-system
+EOF
+    helm init --service-account tiller
+
+    tryuntil "helm ls" helm
+fi
+
+# Install nfs, minio and pachyderm
+helm install --wait --namespace ${NAMESPACE} --name ${NFS_NAME} stable/nfs-server-provisioner
+helm install --wait --namespace ${NAMESPACE} --name ${MINIO_NAME} --set acccessKey=${MINIO_ACCESS_KEY_ID} --set secretKey=${MINIO_SECRET_ACCESS_KEY} stable/minio
+helm install --wait --namespace ${NAMESPACE} --name ${PACHYDERM_NAME} stable/pachyderm
+
+# Wait for all pods to be running.
+tryuntil '! kubectl get pods -n '${NAMESPACE}' | grep -v NAME | grep -v Running' bootstrap 1000
+
+# Bootstrap pachyderm and minio with data
+helm install --name bootstrap /go/src/github.com/IntelAI/vck/helm-charts/bootstrap \
+    --set namespace="${NAMESPACE}" \
+    --set minio.bootstrap=true \
+    --set minio.server_address="http://${MINIO_NAME}.${NAMESPACE}.svc:9000" \
+    --set minio.access_key="${MINIO_ACCESS_KEY_ID}" \
+    --set minio.secret_key="${MINIO_SECRET_ACCESS_KEY}" \
+    --set pachyderm.bootstrap=true \
+    --set pachyderm.address="pachd.${NAMESPACE}.svc:650"
+
+# Wait for the pods to run to completion
+tryuntil 'kubectl get pod minio-bootstrap -n '${NAMESPACE}' | grep Completed' "minio bootstrap"
+
+kubectl delete pod minio-bootstrap -n "${NAMESPACE}" || true
+
+tryuntil 'kubectl get pod pachyderm-bootstrap -n '${NAMESPACE}' | grep Completed' "pachyderm bootstrap"
+
+kubectl delete pod pachyderm-bootstrap -n "${NAMESPACE}" || true
diff --git a/test/e2e/vck_test.go b/test/e2e/vck_test.go
@@ -43,9 +43,10 @@ import (
 )
 
 var (
-	namespace   = flag.String("namespace", "e2e-test", "namespace used for the e2e test")
-	s3ServerIP  = flag.String("s3serverip", "", "S3 server IP address")
-	nfsServerIP = flag.String("nfsserverip", "", "NFS server IP address")
+	namespace        = flag.String("namespace", "e2e-test", "namespace used for the e2e test")
+	s3ServerIP       = flag.String("s3serverip", "", "S3 server IP address")
+	nfsServerIP      = flag.String("nfsserverip", "", "NFS server IP address")
+	pachydermAddress = flag.String("pachydermaddress", "pachd.default.svc:650", "Pachyderm service address")
 )
 
 func makeClients(t *testing.T) (crv1alpha1_volume_manager.VolumeManagerInterface, *kubernetes.Clientset) {
@@ -239,10 +240,11 @@ func TestVolumeManager(t *testing.T) {
 						"key2": "val2",
 					},
 					Options: map[string]string{
-						"repo":       "test",
-						"branch":     "master",
-						"inputPath":  "s3/test",
-						"outputPath": "test",
+						"repo":                    "test",
+						"branch":                  "master",
+						"inputPath":               "s3/test",
+						"outputPath":              "test",
+						"pachydermServiceAddress": *pachydermAddress,
 					},
 				},
 			},
@@ -266,10 +268,11 @@ func TestVolumeManager(t *testing.T) {
 						"key2": "val2",
 					},
 					Options: map[string]string{
-						"repo":       "test",
-						"branch":     "master",
-						"inputPath":  "s3/",
-						"outputPath": "test",
+						"repo":                    "test",
+						"branch":                  "master",
+						"inputPath":               "s3/",
+						"outputPath":              "test",
+						"pachydermServiceAddress": *pachydermAddress,
 					},
 				},
 			},
@@ -488,38 +491,10 @@ func TestVolumeManager(t *testing.T) {
 			expNA:      false,
 			expPVC:     false,
 		},
-		{
-			description: "single vc - Pachyderm - ",
-			volumeConfigs: []crv1alpha1.VolumeConfig{
-				{
-					ID:         "vol1",
-					Replicas:   1,
-					SourceType: "Pachyderm",
-					AccessMode: "ReadWriteOnce",
-					Capacity:   "5Gi",
-					Labels: map[string]string{
-						"key1": "val1",
-						"key2": "val2",
-					},
-					Options: map[string]string{
-						"repo":                   "test",
-						"branch":                 "master",
-						"inputPath":              "s3/",
-						"outputPath":             "test",
-						"timeoutForDataDownload": "10s",
-					},
-				},
-			},
-			expSuccess: true,
-			expError:   "",
-			expHP:      true,
-			expNA:      true,
-			expPVC:     false,
-		},
 	}
 
 	for _, testCase := range testCases {
-		fmt.Printf("%s", testCase.description)
+		fmt.Printf("%s\n", testCase.description)
 		volman := makeVolumeManager(testCase.volumeConfigs)
 		createdVolman, err := crdClient.Create(volman)
 		require.Nil(t, err)