🏃 Add Zuul jobs for e2e conformance tests

.zuul.yaml

@@ -0,0 +1,28 @@
+- project:
+    name: kubernetes-sigs/cluster-api-provider-openstack
+    check:
+      jobs:
+        - cluster-api-provider-openstack-current-acceptance-test-v1.18
+
+- job:
+    name: cluster-api-provider-openstack-conformance-pr
+    parent: init-test
+    timeout: 36000
+    description: |
+      Test CAPO & Kubernetes
+    run: .zuul/playbooks/run.yaml
+    nodeset: ubuntu-bionic-large
+    vars:
+      go_version: '1.13.8'
+      k8s_log_dir: '{{ ansible_user_dir }}/workspace/logs/_artifacts'
+      k8s_os_capi_provider_src_dir: '{{ ansible_user_dir }}/src/sigs.k8s.io/cluster-api-provider-openstack'
+      extra_args: ''
+
+- job:
+    name: cluster-api-provider-openstack-current-acceptance-test-v1.18
+    parent: cluster-api-provider-openstack-conformance-pr
+    description: |
+      Test CAPO current against Kubernetes release-1.18
+    vars:
+      k8s_version_series: '1.18'
+      extra_args: '--use-ci-artifacts'

.zuul/playbooks/run.yaml

@@ -0,0 +1,90 @@
+- hosts: all
+  become: yes
+  pre_tasks:
+  - name: Enable nested virt
+    shell:
+      cmd: |
+        # enable nested virt
+        cat /sys/module/kvm_intel/parameters/nested
+        sudo rmmod kvm-intel
+        sudo sh -c "echo 'options kvm-intel nested=y' >> /etc/modprobe.d/dist.conf"
+        sudo modprobe kvm-intel
+        cat /sys/module/kvm_intel/parameters/nested
+        modinfo kvm_intel | grep nested
+  roles:
+  - move-k8s-repo-to-k8s-specific-dir
+  - config-golang
+  - clone-devstack-gate-to-workspace
+  - role: create-devstack-local-conf
+    enable_services:
+    - 'nova'
+    - 'neutron'
+    - 'keystone'
+    - 'glance'
+  - role: install-devstack
+    environment:
+      USE_PYTHON3: 'True'
+      DEVSTACK_GATE_LIBVIRT_TYPE: 'kvm'
+      OS_BRANCH: 'stable/train'
+      OVERRIDE_ENABLED_SERVICES: 'dstat,etcd3,g-api,g-reg,key,mysql,n-api,n-api-meta,n-cond,n-cpu,n-novnc,n-sch,placement-api,q-agt,q-dhcp,q-l3,q-meta,q-metering,q-svc,rabbit'
+  tasks:
+  - name: Run kubernetes E2E conformance tests with ClusterAPI OpenStack
+    shell:
+      cmd: |
+        set -xeo pipefail
+
+        source /opt/stack/new/devstack/openrc admin admin
+
+        # Print some infos
+        nova hypervisor-stats
+        openstack host list
+        openstack usage list
+        openstack project list
+        openstack network list
+        openstack subnet list
+        openstack image list
+        openstack flavor list
+        openstack server list
+        openstack availability zone list
+        openstack domain list
+
+        openstack flavor delete m1.small
+        openstack flavor create --ram 4096 --disk 10 --vcpus 2 --public --id 2 m1.small --property hw_rng:allowed='True'
+        openstack flavor delete m1.medium
+        openstack flavor create --ram 6144 --disk 10 --vcpus 4 --public --id 3 m1.medium --property hw_rng:allowed='True'
+
+        #  Switch to demo project
+        source /opt/stack/new/devstack/openrc admin demo
+
+        # Create clouds.yaml
+        cat << EOF >> /tmp/clouds.yaml
+        clouds:
+          capi-quickstart:
+            auth:
+              username: ${OS_USERNAME}
+              password: ${OS_PASSWORD}
+              user_domain_id: ${OS_USER_DOMAIN_NAME}
+              auth_url: ${OS_AUTH_URL}
+              domain_id: default
+              project_name: demo
+            verify: false
+            region_name: RegionOne
+        EOF
+        cat /tmp/clouds.yaml
+
+        export ARTIFACTS=/home/zuul/workspace/logs/_artifacts
+        export KUBERNETES_VERSION_SERIES={{ k8s_version_series }}
+        export OPENSTACK_CONTROLPLANE_IP="172.24.5.24"
+        export OPENSTACK_DNS_NAMESERVERS=8.8.8.8
+        export CONTROL_PLANE_MACHINE_COUNT=1
+        export WORKER_MACHINE_COUNT=1
+
+        ./hack/ci/e2e-conformance.sh --install-prereqs --run-tests-parallel --delete-cluster {{ extra_args }}
+
+        # Print some infos
+        ps aux
+        df -h
+
+      executable: /bin/bash
+      chdir: '{{ k8s_os_capi_provider_src_dir }}'
+    environment: '{{ global_env }}'

Makefile

@@ -301,7 +301,7 @@ OPENSTACK_EXTERNAL_NETWORK_ID ?= ""
 OPENSTACK_DNS_NAMESERVERS ?= ""
 OPENSTACK_IMAGE_NAME ?= "ubuntu-1910-kube-v1.17.3"
 OPENSTACK_SSH_AUTHORIZED_KEY ?= ""
-OPENSTACK_NODE_MACHINE_FLAVOR ?= "m1.medium"
+OPENSTACK_NODE_MACHINE_FLAVOR ?= "m1.small"
 OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR ?= "m1.medium"
 CLUSTER_NAME ?= "capi-quickstart"
 OPENSTACK_CLUSTER_TEMPLATE ?= "./templates/cluster-template-without-lb.yaml"
@@ -311,7 +311,7 @@ WORKER_MACHINE_COUNT ?= "3"
 LOAD_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG)
 
 .PHONY: create-cluster
-create-cluster: $(CLUSTERCTL) $(ENVSUBST) ## Create a development Kubernetes cluster on OpenStack in a KIND management cluster.
+create-cluster: $(CLUSTERCTL) $(KUSTOMIZE) $(ENVSUBST) ## Create a development Kubernetes cluster on OpenStack in a KIND management cluster.
 
 	# Create clusterctl.yaml to use local OpenStack provider
 	mkdir -p ./out/infrastructure-openstack/v0.3.0
@@ -339,7 +339,7 @@ create-cluster: $(CLUSTERCTL) $(ENVSUBST) ## Create a development Kubernetes clu
 
 	# (Re-)deploy CAPO provider
 	MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) $(MAKE) set-manifest-image
-	kustomize build config > ./out/infrastructure-openstack/v0.3.0/infrastructure-components.yaml
+	$(KUSTOMIZE) build config > ./out/infrastructure-openstack/v0.3.0/infrastructure-components.yaml
 	$(CLUSTERCTL) delete --infrastructure openstack --include-namespace --namespace capo-system || true
 	kubectl wait --for=delete ns/capo-system || true
 	$(CLUSTERCTL) init --config ./out/clusterctl.yaml --infrastructure openstack
@@ -356,6 +356,10 @@ create-cluster: $(CLUSTERCTL) $(ENVSUBST) ## Create a development Kubernetes clu
 	kubectl wait --for condition=established --timeout=60s crds/openstackmachinetemplates.infrastructure.cluster.x-k8s.io
 	kubectl wait --for condition=established --timeout=60s crds/openstackclusters.infrastructure.cluster.x-k8s.io
 
+    # Wait until everything is really ready, as we had some problems with pods being ready but not yet
+    # available when deploying the cluster.
+	sleep 5
+
 	# Create Cluster.
 	kubectl create ns $(CLUSTER_NAME) || true
 	PULL_POLICY=$(PULL_POLICY) \
@@ -375,19 +379,41 @@ create-cluster: $(CLUSTERCTL) $(ENVSUBST) ## Create a development Kubernetes clu
 	    --from=$(OPENSTACK_CLUSTER_TEMPLATE) \
 	    --kubernetes-version $(KUBERNETES_VERSION) \
 	    --control-plane-machine-count=$(CONTROL_PLANE_MACHINE_COUNT) \
-	    --worker-machine-count=$(WORKER_MACHINE_COUNT) \
-	    --target-namespace=$(CLUSTER_NAME) | kubectl apply -f -
+	    --worker-machine-count=$(WORKER_MACHINE_COUNT) > ./hack/ci/e2e-conformance/cluster.yaml
+
+    # Patch Kubernetes version
+	cat ./hack/ci/e2e-conformance/e2e-conformance_patch.yaml.tpl | \
+	  sed "s|\$${OPENSTACK_CLOUD_PROVIDER_CONF_B64}|$(OPENSTACK_CLOUD_PROVIDER_CONF_B64)|" | \
+	  sed "s|\$${OPENSTACK_CLOUD_CACERT_B64}|$(OPENSTACK_CLOUD_CACERT_B64)|" | \
+	  sed "s|\$${KUBERNETES_VERSION}|$(KUBERNETES_VERSION)|" | \
+	  sed "s|\$${CLUSTER_NAME}|$(CLUSTER_NAME)|"  \
+	   > ./hack/ci/e2e-conformance/e2e-conformance_patch.yaml
+	$(KUSTOMIZE) build --reorder=none hack/ci/e2e-conformance  > ./out/cluster.yaml
+
+	# Deploy cluster
+	kubectl apply -f ./out/cluster.yaml
 
 	# Wait for the kubeconfig to become available.
-	timeout 300 bash -c "while ! kubectl -n $(CLUSTER_NAME) get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
+	timeout 300 bash -c "while ! kubectl get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 10; done"
 	# Get kubeconfig and store it locally.
-	kubectl -n $(CLUSTER_NAME) get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig
-	timeout 900 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 1; done"
+	kubectl get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig
+	timeout 900 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 10; done"
 
 	# Deploy calico
 	curl https://docs.projectcalico.org/manifests/calico.yaml | sed "s/veth_mtu:.*/veth_mtu: \"1400\"/g" | \
 		kubectl --kubeconfig=./kubeconfig apply -f -
 
+.PHONY: delete-cluster
+delete-cluster:
+	kubectl delete cluster --all --ignore-not-found
+
+	kubectl get machinedeployment,kubeadmcontrolplane,cluster
+
+	@if [[ `kubectl get machinedeployment,kubeadmcontrolplane,cluster | wc -l` -gt 0 ]]; then \
+	  echo "Error: not all resources have been deleted correctly"; \
+	  exit 1; \
+	fi
+
 .PHONY: kind-reset
 kind-reset: ## Destroys the "clusterapi" kind cluster.
 	kind delete cluster --name=clusterapi || true