From 1100614d2b08b0a14c6b8de62e0f9ee13aed3ff5 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 26 Aug 2019 10:06:46 -0700 Subject: [PATCH] Implement test framework to test orchestration --- Makefile | 3 +- tester/cmd/main.go | 17 +++ tester/framework.go | 105 ++++++++++++++++ tester/pkg/cluster.go | 35 ++++++ tester/pkg/config.go | 45 +++++++ tester/pkg/kops.go | 258 ++++++++++++++++++++++++++++++++++++++++ tester/pkg/steps.go | 48 ++++++++ tester/test-config.yaml | 104 ++++++++++++++++ 8 files changed, 614 insertions(+), 1 deletion(-) create mode 100644 tester/cmd/main.go create mode 100644 tester/framework.go create mode 100644 tester/pkg/cluster.go create mode 100644 tester/pkg/config.go create mode 100644 tester/pkg/kops.go create mode 100644 tester/pkg/steps.go create mode 100644 tester/test-config.yaml diff --git a/Makefile b/Makefile index 6bfd864b63..db06e45fa2 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,8 @@ test-integration: .PHONY: test-e2e-single-az test-e2e-single-az: - AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" ./hack/run-e2e-test + #AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" ./hack/run-e2e-test + TESTCONFIG=./tester/test-config.yaml go run tester/cmd/main.go .PHONY: test-e2e-multi-az test-e2e-multi-az: diff --git a/tester/cmd/main.go b/tester/cmd/main.go new file mode 100644 index 0000000000..3f66a1f401 --- /dev/null +++ b/tester/cmd/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "fmt" + "os" + + "github.com/kubernetes-sigs/aws-ebs-csi-driver/tester" +) + +func main() { + test := tester.NewTester("") + err := test.Start() + if err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/tester/framework.go b/tester/framework.go new file mode 100644 index 0000000000..903ce64d73 --- /dev/null +++ b/tester/framework.go @@ -0,0 +1,105 @@ +package tester + +import ( + "fmt" + "os" + + "github.com/kubernetes-sigs/aws-ebs-csi-driver/tester/pkg" + yaml "gopkg.in/yaml.v2" + "k8s.io/apimachinery/pkg/util/rand" +) + +type Tester struct { + init pkg.Step + build pkg.Step + up pkg.Step + install pkg.Step + test pkg.Step + uninstall pkg.Step + tearDown pkg.Step +} + +func NewTester(configPath string) *Tester { + if configPath == "" { + configPath = readTestConfigPath() + } + testConfig, err := os.Open(configPath) + if err != nil { + panic(err) + } + var config *pkg.TestConfig + err = yaml.NewDecoder(testConfig).Decode(&config) + if err != nil { + panic(err) + } + + testId := fmt.Sprintf("%d", rand.Int()%10000) + clusterCreator, err := pkg.NewClusterCreator(config, "/tmp/ebs-e2e-test", testId) + if err != nil { + panic(err) + } + + return &Tester{ + init: createStepOrPanic(clusterCreator.Init), + build: scriptStep(config.BuildScript, testId), + up: createStepOrPanic(clusterCreator.Up), + install: scriptStep(config.InstallScript, testId), + test: scriptStep(config.TestScript, testId), + uninstall: scriptStep(config.UninstallScript, testId), + tearDown: createStepOrPanic(clusterCreator.TearDown), + } +} + +func (t *Tester) Start() error { + var err error + err = t.init.Run() + if err != nil { + return err + } + + err = t.build.Run() + if err != nil { + return err + } + + err = t.up.Run() + if err != nil { + return err + } + + err = t.install.Run() + if err != nil { + tErr := t.tearDown.Run() + if tErr != nil { + fmt.Printf("failed to tear down cluster: %v", tErr) + } + return err + } + + err = t.test.Run() + t.uninstall.Run() + t.tearDown.Run() + + return err +} + +func readTestConfigPath() string { + path := os.Getenv("TESTCONFIG") + if len(path) == 0 { + return "test-config.yaml" + } + + return path +} + +func createStepOrPanic(f func() (pkg.Step, error)) pkg.Step { + step, err := f() + if err != nil { + panic(err) + } + return step +} + +func scriptStep(script string, testId string) pkg.Step { + return &pkg.TestStep{Script: script, TestId: testId} +} diff --git a/tester/pkg/cluster.go b/tester/pkg/cluster.go new file mode 100644 index 0000000000..ccfd3e42d3 --- /dev/null +++ b/tester/pkg/cluster.go @@ -0,0 +1,35 @@ +package pkg + +import ( + "fmt" +) + +type ClusterCreator interface { + // Initialize the creator such as downloading dependencies + Init() (Step, error) + + // Create and wait for cluster creation + Up() (Step, error) + + // Teardown the cluster + TearDown() (Step, error) +} + +func NewClusterCreator(config *TestConfig, testDir string, testId string) (ClusterCreator, error) { + cluster := config.Cluster + + if cluster.Kops == nil && cluster.Eks == nil { + return nil, fmt.Errorf("TestConfig.Cluster is not set") + } + + if cluster.Kops != nil && cluster.Eks != nil { + return nil, fmt.Errorf("Both Kops and Eks cluster is set") + } + + if cluster.Kops != nil { + return NewKopsClusterCreator(cluster.Kops, testDir, testId), nil + } + + // TODO: add for EKS cluster + return nil, nil +} diff --git a/tester/pkg/config.go b/tester/pkg/config.go new file mode 100644 index 0000000000..6cf87e2e0d --- /dev/null +++ b/tester/pkg/config.go @@ -0,0 +1,45 @@ +package pkg + +type TestConfig struct { + Cluster *Cluster `yaml:"cluster"` + Region string `yaml:"region"` + BuildScript string `yaml:"build"` + InstallScript string `yaml:"install"` + UninstallScript string `yaml:"uninstall"` + TestScript string `yaml:"test"` +} + +type Cluster struct { + Kops *KopsCluster `yaml:"kops"` + Eks *EksCluster `yaml:"eks"` +} + +type KopsCluster struct { + Region string `yaml:"region"` + Zones string `yaml:"zones"` + NodeCount int `yaml:"nodeCount"` + NodeSize string `yaml:"nodeSize"` + KubernetesVersion string `yaml:"kubernetesVersion"` + FeatureGates string `yaml:"featureGates"` + IamPolicies string `yaml:"iamPolicies"` + //KubeApiServer KubeApiServer `yaml:"kubeApiServer"` + //Kubelet Kubelet `yaml:"kubelet"` +} + +type KubeApiServer struct { + FeatureGates FeatureGates `yaml:"featureGates"` +} + +type FeatureGates struct { + CSIDriverRegistry bool `yaml:"CSIDriverRegistry"` + CSINodeInfo bool `yaml:"CSINodeInfo"` + CSIBlockVolume bool `yaml:"CSIBlockVolume"` + VolumeSnapshotDataSource bool `yaml:"VolumeSnapshotDataSource"` +} + +type Kubelet struct { + FeatureGates FeatureGates +} + +type EksCluster struct { +} diff --git a/tester/pkg/kops.go b/tester/pkg/kops.go new file mode 100644 index 0000000000..b257fb7467 --- /dev/null +++ b/tester/pkg/kops.go @@ -0,0 +1,258 @@ +package pkg + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "time" +) + +const ( + kopsStateFile = "s3://k8s-kops-csi-e2e" +) + +type FuncStep struct { + f func() error +} + +func (s *FuncStep) Run() error { + return s.f() +} + +func EmptyStep() Step { + return &FuncStep{func() error { + return nil + }} +} + +type KopsClusterCreator struct { + // TestId is used as cluster name + TestId string + + // Kops represents the configuration to create kops cluster + Kops *KopsCluster + + // directory where tempory data is saved + TestDir string + + // The path to Kops executable + KopsBinaryPath string +} + +func NewKopsClusterCreator(kops *KopsCluster, dir string, testId string) *KopsClusterCreator { + binaryFilePath := filepath.Join(dir, "kops") + return &KopsClusterCreator{ + Kops: kops, + TestDir: dir, + TestId: testId, + KopsBinaryPath: binaryFilePath, + } +} + +func (c *KopsClusterCreator) Init() (Step, error) { + f := func() error { + _, err := os.Stat(c.TestDir) + if os.IsNotExist(err) { + err := os.Mkdir(c.TestDir, 0777) + if err != nil { + return err + } + } + + _, err = os.Stat(c.KopsBinaryPath) + if os.IsNotExist(err) { + return c.downloadKops() + } + + return nil + } + + return &FuncStep{f}, nil +} + +func (c *KopsClusterCreator) Up() (Step, error) { + f := func() error { + // create cluster + err := c.createCluster() + if err != nil { + return err + } + + c.waitForCreation() + + // wait for cluster creation to success + // or return err if failed + return nil + } + + return &FuncStep{f}, nil +} + +func (c *KopsClusterCreator) TearDown() (Step, error) { + + f := func() error { + clusterName := c.clusterName() + log.Printf("Deleting cluster %s", clusterName) + + cmd := exec.Command(c.KopsBinaryPath, "delete", "cluster", + "--state", kopsStateFile, + "--name", clusterName, "--yes") + + cmd.Stdout = os.Stdout + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + + return cmd.Run() + } + + return &FuncStep{f}, nil +} + +func (c *KopsClusterCreator) clusterName() string { + return fmt.Sprintf("test-cluster-%s.k8s.local", c.TestId) +} + +func (c *KopsClusterCreator) downloadKops() error { + osArch := fmt.Sprintf("%s-amd64", runtime.GOOS) + url := fmt.Sprintf("https://github.com/kubernetes/kops/releases/download/1.14.0-alpha.1/kops-%s", osArch) + log.Printf("Downloading KOPS from %s to %s", url, c.KopsBinaryPath) + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + payload, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + err = ioutil.WriteFile(c.KopsBinaryPath, payload, 0777) + if err != nil { + return err + } + + return nil +} + +func (c *KopsClusterCreator) createCluster() error { + clusterName := c.clusterName() + log.Printf("Creating Kops cluster %s", clusterName) + + sshKeyPath := filepath.Join(c.TestDir, "id_rsa") + + _, err := os.Stat(sshKeyPath) + // only generate SSH key if it is missing + if os.IsNotExist(err) { + err := c.generateSSHKey(sshKeyPath) + if err != nil { + return err + } + } + + cmd := exec.Command(c.KopsBinaryPath, "create", "cluster", + "--state", kopsStateFile, + "--zones", c.Kops.Zones, + "--node-count", fmt.Sprintf("%d", c.Kops.NodeCount), + "--node-size", c.Kops.NodeSize, + "--kubernetes-version", c.Kops.KubernetesVersion, + "--ssh-public-key", fmt.Sprintf("%s.pub", sshKeyPath), + clusterName, + ) + cmd.Stdout = os.Stdout + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + + err = cmd.Run() + if err != nil { + return err + } + clusterYamlPath := filepath.Join(c.TestDir, fmt.Sprintf("%s.yaml", c.TestId)) + + clusterYamlFile, err := os.Create(clusterYamlPath) + if err != nil { + return err + } + + cmd = exec.Command(c.KopsBinaryPath, "get", "cluster", + "--state", kopsStateFile, clusterName, "-o", "yaml") + cmd.Stdout = clusterYamlFile + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + clusterYamlFile.Close() + return err + } + + _, err = clusterYamlFile.WriteString(c.Kops.FeatureGates) + if err != nil { + clusterYamlFile.Close() + return err + } + _, err = clusterYamlFile.WriteString(c.Kops.IamPolicies) + if err != nil { + clusterYamlFile.Close() + return err + } + err = clusterYamlFile.Sync() + if err != nil { + clusterYamlFile.Close() + return err + } + clusterYamlFile.Close() + + cmd = exec.Command(c.KopsBinaryPath, "replace", + "--state", kopsStateFile, "-f", clusterYamlPath) + cmd.Stdout = os.Stdout + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + + err = cmd.Run() + if err != nil { + return err + } + + cmd = exec.Command(c.KopsBinaryPath, "update", "cluster", + "--state", kopsStateFile, clusterName, "--yes") + cmd.Stdout = os.Stdout + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + + err = cmd.Run() + if err != nil { + return err + } + + return nil +} + +func (c *KopsClusterCreator) waitForCreation() { + for { + cmd := exec.Command(c.KopsBinaryPath, "validate", "cluster", + "--state", kopsStateFile) + cmd.Stdout = os.Stdout + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + err := cmd.Run() + if err == nil { + break + } + time.Sleep(30 * time.Second) + } +} + +func (c *KopsClusterCreator) generateSSHKey(keyPath string) error { + cmd := exec.Command("ssh-keygen", "-N", "", "-f", keyPath) + + cmd.Stdout = os.Stdout + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + + return cmd.Run() +} diff --git a/tester/pkg/steps.go b/tester/pkg/steps.go new file mode 100644 index 0000000000..bd3fb04793 --- /dev/null +++ b/tester/pkg/steps.go @@ -0,0 +1,48 @@ +package pkg + +import ( + "os" + "os/exec" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sts" +) + +type Step interface { + Run() error +} + +type TestStep struct { + Script string + TestId string +} + +func (s *TestStep) Run() error { + script := strings.Replace(s.Script, "{{TEST_ID}}", s.TestId, -1) + testCmd := exec.Command("sh", "-c", script) + testCmd.Stdout = os.Stdout + testCmd.Stdin = os.Stdin + testCmd.Stderr = os.Stderr + err := testCmd.Run() + if err != nil { + return err + } + return nil +} + +func getAwsAccountId(region string) (string, error) { + awsConfig := &aws.Config{ + Region: aws.String(region), + } + svc := sts.New(session.New(awsConfig)) + input := &sts.GetCallerIdentityInput{} + + result, err := svc.GetCallerIdentity(input) + if err != nil { + return "", err + } + + return aws.StringValue(result.Account), nil +} diff --git a/tester/test-config.yaml b/tester/test-config.yaml new file mode 100644 index 0000000000..9e729e246c --- /dev/null +++ b/tester/test-config.yaml @@ -0,0 +1,104 @@ +cluster: + kops: + zones: us-west-2a,us-west-2b,us-west-2c + nodeCount: 3 + nodeSize: c5.large + kubernetesVersion: 1.15.3 + featureGates: |2 + kubeAPIServer: + featureGates: + CSIDriverRegistry: "true" + CSINodeInfo: "true" + CSIBlockVolume: "true" + CSIMigration: "true" + CSIMigrationAWS: "true" + ExpandCSIVolumes: "true" + VolumeSnapshotDataSource: "true" + CSIInlineVolume: "true" + kubeControllerManager: + featureGates: + CSIDriverRegistry: "true" + CSINodeInfo: "true" + CSIBlockVolume: "true" + CSIMigration: "true" + CSIMigrationAWS: "true" + ExpandCSIVolumes: "true" + CSIInlineVolume: "true" + kubelet: + featureGates: + CSIDriverRegistry: "true" + CSINodeInfo: "true" + CSIBlockVolume: "true" + CSIMigration: "true" + CSIMigrationAWS: "true" + ExpandCSIVolumes: "true" + CSIInlineVolume: "true" + iamPolicies: |2 + additionalPolicies: + node: | + [ + { + "Effect": "Allow", + "Action": [ + "ec2:AttachVolume", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DeleteSnapshot", + "ec2:DeleteTags", + "ec2:DeleteVolume", + "ec2:DescribeInstances", + "ec2:DescribeSnapshots", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DetachVolume", + "ec2:ModifyVolume", + "ec2:DescribeVolumesModifications" + ], + "Resource": "*" + } + ] + +build: | + eval $(aws ecr get-login --region us-west-2 --no-include-email) + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + IMAGE_TAG={{TEST_ID}} + IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-ebs-csi-driver + docker build -t $IMAGE_NAME:$IMAGE_TAG . + docker push $IMAGE_NAME:$IMAGE_TAG + +install: | + echo "Deploying driver" + # install helm + OS_ARCH=$(go env GOOS)-amd64 + helm_name=helm-v2.14.1-$OS_ARCH.tar.gz + wget https://get.helm.sh/$helm_name + tar xvzf $helm_name + mv $OS_ARCH/helm /usr/local/bin/helm + + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + IMAGE_TAG={{TEST_ID}} + IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-ebs-csi-driver + + # install tiller + kubectl apply -f ./hack/utils/tiller-rbac.yaml + helm init --service-account tiller --history-max 200 --wait + kubectl get po -n kube-system + + helm install --name aws-ebs-csi-driver \ + --set enableVolumeScheduling=true \ + --set enableVolumeResizing=true \ + --set enableVolumeSnapshot=true \ + --set image.repository=$IMAGE_NAME \ + --set image.tag=$IMAGE_TAG \ + ./aws-ebs-csi-driver + +uninstall: | + echo "Removing driver" + helm del --purge aws-ebs-csi-driver + +test: | + go get -u github.com/onsi/ginkgo/ginkgo + export KUBECONFIG=$HOME/.kube/config + export AWS_AVAILABILITY_ZONES=us-west-2a + ginkgo -p -nodes=32 -v --focus="\[ebs-csi-e2e\] \[single-az\]" tests/e2e -- -report-dir=$ARTIFACTS