Use new test framework for test orchestration

Makefile

@@ -46,15 +46,17 @@ test-integration:
 
 .PHONY: test-e2e-single-az
 test-e2e-single-az:
-	AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" ./hack/run-e2e-test
+	TESTCONFIG=./tester/single-az-config.yaml go run tester/cmd/main.go
 
 .PHONY: test-e2e-multi-az
 test-e2e-multi-az:
-	AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c INSTANCE_TYPE=c5.large GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" ./hack/run-e2e-test
+	TESTCONFIG=./tester/multi-az-config.yaml go run tester/cmd/main.go
 
 .PHONY: test-e2e-migration
 test-e2e-migration:
 	AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-migration\]" ./hack/run-e2e-test
+	# TODO: enable migration test to use new framework
+	#TESTCONFIG=./tester/migration-test-config.yaml go run tester/cmd/main.go
 
 .PHONY: image-release
 image-release:

go.mod

@@ -4,6 +4,7 @@ require (
 	bitbucket.org/ww/goautoneg v0.0.0-20120707110453-75cd24fc2f2c // indirect
 	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/NYTimes/gziphandler v1.0.1 // indirect
+	github.com/aws/aws-k8s-tester/e2e/tester v0.0.0-20190906052806-9cfd86ce7f9c // indirect
 	github.com/aws/aws-sdk-go v1.16.26
 	github.com/container-storage-interface/spec v1.1.0
 	github.com/coreos/go-semver v0.2.0 // indirect
@@ -37,9 +38,6 @@ require (
 	github.com/json-iterator/go v1.1.5 // indirect
 	github.com/kubernetes-csi/csi-test v2.0.0+incompatible
 	github.com/kubernetes-csi/external-snapshotter v1.0.1
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/onsi/ginkgo v1.8.0
 	github.com/onsi/gomega v1.4.3
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
@@ -63,7 +61,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/square/go-jose.v2 v2.2.1 // indirect
-	gopkg.in/yaml.v2 v2.2.2 // indirect
+	gopkg.in/yaml.v2 v2.2.2
 	k8s.io/api v0.0.0
 	k8s.io/apimachinery v0.0.0
 	k8s.io/client-go v0.0.0

tester/cmd/main.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/aws/aws-k8s-tester/e2e/tester"
+)
+
+func main() {
+	tester.Start()
+}

tester/migration-test-config.yaml

@@ -0,0 +1,136 @@
+cluster:
+  kops:
+    stateFile: s3://k8s-kops-csi-e2e
+    zones: us-west-2a
+    nodeCount: 3
+    nodeSize: c5.large
+    kubernetesVersion: 1.15.3
+    featureGates: |2
+        kubeAPIServer:
+          featureGates:
+            CSIDriverRegistry: "true"
+            CSINodeInfo: "true"
+            CSIBlockVolume: "true"
+            CSIMigration: "true"
+            CSIMigrationAWS: "true"
+            ExpandCSIVolumes: "true"
+            VolumeSnapshotDataSource: "true"
+            CSIInlineVolume: "true"
+        kubeControllerManager:
+          featureGates:
+            CSIDriverRegistry: "true"
+            CSINodeInfo: "true"
+            CSIBlockVolume: "true"
+            CSIMigration: "true"
+            CSIMigrationAWS: "true"
+            ExpandCSIVolumes: "true"
+            CSIInlineVolume: "true"
+        kubelet:
+          featureGates:
+            CSIDriverRegistry: "true"
+            CSINodeInfo: "true"
+            CSIBlockVolume: "true"
+            CSIMigration: "true"
+            CSIMigrationAWS: "true"
+            ExpandCSIVolumes: "true"
+            CSIInlineVolume: "true"
+    iamPolicies: |2
+        additionalPolicies:
+          node: |
+            [
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "ec2:AttachVolume",
+                  "ec2:CreateSnapshot",
+                  "ec2:CreateTags",
+                  "ec2:CreateVolume",
+                  "ec2:DeleteSnapshot",
+                  "ec2:DeleteTags",
+                  "ec2:DeleteVolume",
+                  "ec2:DescribeInstances",
+                  "ec2:DescribeSnapshots",
+                  "ec2:DescribeTags",
+                  "ec2:DescribeVolumes",
+                  "ec2:DetachVolume",
+                  "ec2:ModifyVolume",
+                  "ec2:DescribeVolumesModifications"
+                ],
+                "Resource": "*"
+              }
+            ]
+
+build: |
+  eval $(aws ecr get-login --region us-west-2 --no-include-email)
+  AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+  IMAGE_TAG={{TEST_ID}}
+  IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-ebs-csi-driver
+  docker build -t $IMAGE_NAME:$IMAGE_TAG .
+  docker push $IMAGE_NAME:$IMAGE_TAG
+
+install: |
+  echo "Deploying driver"
+  # install helm
+  OS_ARCH=$(go env GOOS)-amd64
+  helm_name=helm-v2.14.1-$OS_ARCH.tar.gz
+  wget https://get.helm.sh/$helm_name
+  tar xvzf $helm_name
+  mv $OS_ARCH/helm /usr/local/bin/helm
+
+  AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+  IMAGE_TAG={{TEST_ID}}
+  IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-ebs-csi-driver
+
+  # install tiller
+  kubectl apply -f ./hack/utils/tiller-rbac.yaml 
+  helm init --service-account tiller --history-max 200 --wait
+  kubectl get po -n kube-system
+
+  helm install --name aws-ebs-csi-driver \
+      --set enableVolumeScheduling=true \
+      --set enableVolumeResizing=true \
+      --set enableVolumeSnapshot=true \
+      --set image.repository=$IMAGE_NAME \
+      --set image.tag=$IMAGE_TAG \
+      ./aws-ebs-csi-driver
+
+uninstall: | 
+  echo "Removing driver"
+  helm del --purge aws-ebs-csi-driver
+
+test: |
+  # TODO known test failures to skip temporarily
+  # - should not allow expansion of pvcs without AllowVolumeExpansion property
+  #   - Test passes but cleanup fails, need https://github.com/kubernetes/kubernetes/pull/81107
+  # - (block volmode) Verify if offline PVC expansion works / should resize volume when PVC is edited while pod is using it
+  #   - NodeExpand for BlockVolumes not well-defined, need more investigation and possibly https://github.com/container-storage-interface/spec/issues/380
+  # - should provision storage with mount options
+  #   - Known bug, need https://github.com/kubernetes/kubernetes/pull/80191 but not yet in a patch release
+  cd ./tests/e2e-migration
+  go test -v -timeout 0 ./... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="\[ebs-csi-migration\]" -ginkgo.skip="\[Disruptive\]\
+    |should.not.allow.expansion\
+    |block.volmode.+volume-expand\
+    |should.provision.storage.with.mount.options\
+    |should.not.mount./.map.unused.volumes.in.a.pod" -gce-zone=us-west-2a
+  TEST_PASS=$?
+  cd ../..
+
+  # There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com
+  # Find the controller-manager log and read its metrics to verify
+  NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name")
+  kubectl port-forward kube-controller-manager-$NODE 10252:10252 -n kube-system&
+  curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="ebs.csi.aws.com"'
+  CSI_CALLED=$?
+  curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="kubernetes.io/aws-ebs"'
+  INTREE_CALLED=$?
+
+  # TEST_PASS if tests passed, CSI was called, and In-tree was not called
+  if [ "$TEST_PASS" == 0 ] && [ "$CSI_CALLED" == 0 ] && [ "$INTREE_CALLED" == 1 ]; then
+      TEST_PASS=0
+  else
+      TEST_PASS=1
+  fi
+
+  if [[ $TEST_PASS -ne 0 ]]; then
+      exit 1
+  fi

tester/multi-az-config.yaml

@@ -0,0 +1,105 @@
+cluster:
+  kops:
+    stateFile: s3://k8s-kops-csi-e2e
+    zones: us-west-2a,us-west-2b,us-west-2c
+    nodeCount: 3
+    nodeSize: c5.large
+    kubernetesVersion: 1.15.3
+    featureGates: |2
+        kubeAPIServer:
+          featureGates:
+            CSIDriverRegistry: "true"
+            CSINodeInfo: "true"
+            CSIBlockVolume: "true"
+            CSIMigration: "true"
+            CSIMigrationAWS: "true"
+            ExpandCSIVolumes: "true"
+            VolumeSnapshotDataSource: "true"
+            CSIInlineVolume: "true"
+        kubeControllerManager:
+          featureGates:
+            CSIDriverRegistry: "true"
+            CSINodeInfo: "true"
+            CSIBlockVolume: "true"
+            CSIMigration: "true"
+            CSIMigrationAWS: "true"
+            ExpandCSIVolumes: "true"
+            CSIInlineVolume: "true"
+        kubelet:
+          featureGates:
+            CSIDriverRegistry: "true"
+            CSINodeInfo: "true"
+            CSIBlockVolume: "true"
+            CSIMigration: "true"
+            CSIMigrationAWS: "true"
+            ExpandCSIVolumes: "true"
+            CSIInlineVolume: "true"
+    iamPolicies: |2
+        additionalPolicies:
+          node: |
+            [
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "ec2:AttachVolume",
+                  "ec2:CreateSnapshot",
+                  "ec2:CreateTags",
+                  "ec2:CreateVolume",
+                  "ec2:DeleteSnapshot",
+                  "ec2:DeleteTags",
+                  "ec2:DeleteVolume",
+                  "ec2:DescribeInstances",
+                  "ec2:DescribeSnapshots",
+                  "ec2:DescribeTags",
+                  "ec2:DescribeVolumes",
+                  "ec2:DetachVolume",
+                  "ec2:ModifyVolume",
+                  "ec2:DescribeVolumesModifications"
+                ],
+                "Resource": "*"
+              }
+            ]
+
+build: |
+  eval $(aws ecr get-login --region us-west-2 --no-include-email)
+  AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+  IMAGE_TAG={{TEST_ID}}
+  IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-ebs-csi-driver
+  docker build -t $IMAGE_NAME:$IMAGE_TAG .
+  docker push $IMAGE_NAME:$IMAGE_TAG
+
+install: |
+  echo "Deploying driver"
+  # install helm
+  OS_ARCH=$(go env GOOS)-amd64
+  helm_name=helm-v2.14.1-$OS_ARCH.tar.gz
+  wget https://get.helm.sh/$helm_name
+  tar xvzf $helm_name
+  mv $OS_ARCH/helm /usr/local/bin/helm
+
+  AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+  IMAGE_TAG={{TEST_ID}}
+  IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-ebs-csi-driver
+
+  # install tiller
+  kubectl apply -f ./hack/utils/tiller-rbac.yaml 
+  helm init --service-account tiller --history-max 200 --wait
+  kubectl get po -n kube-system
+
+  helm install --name aws-ebs-csi-driver \
+      --set enableVolumeScheduling=true \
+      --set enableVolumeResizing=true \
+      --set enableVolumeSnapshot=true \
+      --set image.repository=$IMAGE_NAME \
+      --set image.tag=$IMAGE_TAG \
+      ./aws-ebs-csi-driver
+
+uninstall: | 
+  echo "Removing driver"
+  helm del --purge aws-ebs-csi-driver
+
+test: |
+  go get -u github.com/onsi/ginkgo/ginkgo
+  export KUBECONFIG=$HOME/.kube/config
+  export AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c
+  ginkgo -p -nodes=32 -v --focus="\[ebs-csi-e2e\] \[multi-az\]" tests/e2e -- -report-dir=$ARTIFACTS