From 06f52cfe3e15a633963eb54fc76afaf6620a5f0d Mon Sep 17 00:00:00 2001 From: kairen Date: Fri, 1 Dec 2017 18:40:30 +0800 Subject: [PATCH 1/4] Add integration test for ingress addon --- test/integration/addons_test.go | 48 ++++++++++++++++++++ test/integration/functional_test.go | 1 + test/integration/testdata/nginx-ing.yaml | 15 +++++++ test/integration/util/util.go | 57 ++++++++++++++++++++++++ 4 files changed, 121 insertions(+) create mode 100644 test/integration/testdata/nginx-ing.yaml diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index dfdba1bed1d6..ebac771b4220 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -22,6 +22,7 @@ import ( "fmt" "net" "net/url" + "path/filepath" "strings" "testing" "time" @@ -68,6 +69,53 @@ func testDashboard(t *testing.T) { } } +func testIngressController(t *testing.T) { + t.Parallel() + minikubeRunner := NewMinikubeRunner(t) + kubectlRunner := util.NewKubectlRunner(t) + + minikubeRunner.RunCommand("addons enable ingress", true) + if err := util.WaitForIngressControllerRunning(t); err != nil { + t.Fatalf("waiting for ingress-controller to be up: %s", err) + } + + if err := util.WaitForIngressDefaultBackendRunning(t); err != nil { + t.Fatalf("waiting for default-http-backend to be up: %s", err) + } + + ingressPath, _ := filepath.Abs("testdata/nginx-ing.yaml") + if _, err := kubectlRunner.RunCommand([]string{"create", "-f", ingressPath}); err != nil { + t.Fatalf("creating nginx ingress resource: %s", err) + } + + podName := "nginx" + args := []string{"run", podName, "--image=nginx:alpine", "--port=80", "--restart=Never"} + if _, err := kubectlRunner.RunCommand(args); err != nil { + t.Fatalf("failed to create nginx pods: %s", err) + } + + args = []string{"expose", "pod", podName, "--target-port=80"} + if _, err := kubectlRunner.RunCommand(args); err != nil { + t.Fatalf("failed to create nginx service: %s", err) + } + + if err := util.WaitForNginxRunning(t); err != nil { + t.Fatalf("waiting for nginx to be up: %s", err) + } + + expectedStr := "Welcome to nginx!" + runCmd := fmt.Sprintf("curl http://127.0.0.1:80 -H 'Host: nginx.example.com'") + sshCmdOutput, _ := minikubeRunner.SSH(runCmd) + if !strings.Contains(sshCmdOutput, expectedStr) { + t.Fatalf("ExpectedStr sshCmdOutput to be: %s. Output was: %s", expectedStr, sshCmdOutput) + } + + defer kubectlRunner.RunCommand([]string{"delete", "pod", podName}) + defer kubectlRunner.RunCommand([]string{"delete", "svc", podName}) + defer kubectlRunner.RunCommand([]string{"delete", "-f", ingressPath}) + minikubeRunner.RunCommand("addons disable ingress", true) +} + func testServicesList(t *testing.T) { t.Parallel() minikubeRunner := NewMinikubeRunner(t) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index f9f16b1d5029..a9d28b36b8b3 100755 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -34,6 +34,7 @@ func TestFunctional(t *testing.T) { t.Run("Logs", testClusterLogs) t.Run("Addons", testAddons) t.Run("Dashboard", testDashboard) + t.Run("IngressController", testIngressController) t.Run("ServicesList", testServicesList) t.Run("Provisioning", testProvisioning) diff --git a/test/integration/testdata/nginx-ing.yaml b/test/integration/testdata/nginx-ing.yaml new file mode 100644 index 000000000000..30cef8139da7 --- /dev/null +++ b/test/integration/testdata/nginx-ing.yaml @@ -0,0 +1,15 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: nginx-ingress + labels: + integration-test: ingress +spec: + rules: + - host: nginx.example.com + http: + paths: + - path: / + backend: + serviceName: nginx + servicePort: 80 diff --git a/test/integration/util/util.go b/test/integration/util/util.go index aaf019b4f64b..a7bc3c64b22a 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -263,6 +263,63 @@ func WaitForDashboardRunning(t *testing.T) error { return nil } +func WaitForIngressControllerRunning(t *testing.T) error { + client, err := commonutil.GetClient() + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + + if err := commonutil.WaitForRCToStabilize(client, "kube-system", "nginx-ingress-controller", time.Minute*10); err != nil { + return errors.Wrap(err, "waiting for ingress-controller RC to stabilize") + } + + selector := labels.SelectorFromSet(labels.Set(map[string]string{"app": "nginx-ingress-controller"})) + if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { + return errors.Wrap(err, "waiting for ingress-controller pods") + } + + return nil +} + +func WaitForIngressDefaultBackendRunning(t *testing.T) error { + client, err := commonutil.GetClient() + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + + if err := commonutil.WaitForRCToStabilize(client, "kube-system", "default-http-backend", time.Minute*10); err != nil { + return errors.Wrap(err, "waiting for default-http-backend RC to stabilize") + } + + if err := commonutil.WaitForService(client, "kube-system", "default-http-backend", true, time.Millisecond*500, time.Minute*10); err != nil { + return errors.Wrap(err, "waiting for default-http-backend service to be up") + } + + if err := commonutil.WaitForServiceEndpointsNum(client, "kube-system", "default-http-backend", 1, time.Second*3, time.Minute*10); err != nil { + return errors.Wrap(err, "waiting for one default-http-backend endpoint to be up") + } + + return nil +} + +func WaitForNginxRunning(t *testing.T) error { + client, err := commonutil.GetClient() + + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + + selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"})) + if err := commonutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { + return errors.Wrap(err, "waiting for nginx pods") + } + + if err := commonutil.WaitForService(client, "default", "nginx", true, time.Millisecond*500, time.Minute*10); err != nil { + t.Errorf("Error waiting for nginx service to be up") + } + return nil +} + func Retry(t *testing.T, callback func() error, d time.Duration, attempts int) (err error) { for i := 0; i < attempts; i++ { err = callback() From 1fded4e97e1be41127b7651c35f3fec2cffe7a5a Mon Sep 17 00:00:00 2001 From: kairen Date: Sat, 2 Dec 2017 02:46:52 +0800 Subject: [PATCH 2/4] Copy nginx-ing.yaml to testdata directory on Jenkins --- hack/jenkins/common.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 548bf01b8809..90de77814a93 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -33,6 +33,7 @@ gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/e2e-${OS_ARCH} out/ gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/busybox.yaml testdata/ gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/pvc.yaml testdata/ gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/busybox-mount-test.yaml testdata/ +gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/nginx-ing.yaml testdata/ export MINIKUBE_WANTREPORTERRORPROMPT=False sudo ./out/minikube-${OS_ARCH} delete || true From 745645b92910f2fb49afdd915f0651a66e64ac41 Mon Sep 17 00:00:00 2001 From: kairen Date: Sat, 2 Dec 2017 03:28:47 +0800 Subject: [PATCH 3/4] Use yaml file to create nginx pod and service --- hack/jenkins/common.sh | 1 + test/integration/addons_test.go | 15 +++------- test/integration/testdata/nginx-pod-svc.yaml | 31 ++++++++++++++++++++ 3 files changed, 36 insertions(+), 11 deletions(-) create mode 100644 test/integration/testdata/nginx-pod-svc.yaml diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 90de77814a93..4f232bb7849a 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -33,6 +33,7 @@ gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/e2e-${OS_ARCH} out/ gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/busybox.yaml testdata/ gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/pvc.yaml testdata/ gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/busybox-mount-test.yaml testdata/ +gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/nginx-pod-svc.yaml testdata/ gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/nginx-ing.yaml testdata/ export MINIKUBE_WANTREPORTERRORPROMPT=False diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index ebac771b4220..864ed164a9f9 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -88,15 +88,9 @@ func testIngressController(t *testing.T) { t.Fatalf("creating nginx ingress resource: %s", err) } - podName := "nginx" - args := []string{"run", podName, "--image=nginx:alpine", "--port=80", "--restart=Never"} - if _, err := kubectlRunner.RunCommand(args); err != nil { - t.Fatalf("failed to create nginx pods: %s", err) - } - - args = []string{"expose", "pod", podName, "--target-port=80"} - if _, err := kubectlRunner.RunCommand(args); err != nil { - t.Fatalf("failed to create nginx service: %s", err) + podPath, _ := filepath.Abs("testdata/nginx-pod-svc.yaml") + if _, err := kubectlRunner.RunCommand([]string{"create", "-f", podPath}); err != nil { + t.Fatalf("creating nginx ingress resource: %s", err) } if err := util.WaitForNginxRunning(t); err != nil { @@ -110,8 +104,7 @@ func testIngressController(t *testing.T) { t.Fatalf("ExpectedStr sshCmdOutput to be: %s. Output was: %s", expectedStr, sshCmdOutput) } - defer kubectlRunner.RunCommand([]string{"delete", "pod", podName}) - defer kubectlRunner.RunCommand([]string{"delete", "svc", podName}) + defer kubectlRunner.RunCommand([]string{"delete", "-f", podPath}) defer kubectlRunner.RunCommand([]string{"delete", "-f", ingressPath}) minikubeRunner.RunCommand("addons disable ingress", true) } diff --git a/test/integration/testdata/nginx-pod-svc.yaml b/test/integration/testdata/nginx-pod-svc.yaml new file mode 100644 index 000000000000..67782f275c3c --- /dev/null +++ b/test/integration/testdata/nginx-pod-svc.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + run: nginx + name: nginx + namespace: default +spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - containerPort: 80 + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + run: nginx + name: nginx + namespace: default +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + run: nginx + sessionAffinity: None + type: ClusterIP From e93a2f4254c143aded4056732e8eb247c982a84f Mon Sep 17 00:00:00 2001 From: kairen Date: Sat, 2 Dec 2017 04:04:39 +0800 Subject: [PATCH 4/4] Ignore the ingress test in Linux-None --- test/integration/functional_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index a9d28b36b8b3..c70c30683bb3 100755 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -34,13 +34,13 @@ func TestFunctional(t *testing.T) { t.Run("Logs", testClusterLogs) t.Run("Addons", testAddons) t.Run("Dashboard", testDashboard) - t.Run("IngressController", testIngressController) t.Run("ServicesList", testServicesList) t.Run("Provisioning", testProvisioning) if !strings.Contains(minikubeRunner.StartArgs, "--vm-driver=none") { t.Run("EnvVars", testClusterEnv) t.Run("SSH", testClusterSSH) + t.Run("IngressController", testIngressController) // t.Run("Mounting", testMounting) } }