🏃 Add e2e tests for all flavors

.golangci.yml

@@ -23,7 +23,7 @@ linters:
 linters-settings:
   cyclop:
     # TODO(sbuerin) fix remaining findings and set to 20 afterwards
-    max-complexity: 29
+    max-complexity: 30
   goimports:
     local-prefixes: sigs.k8s.io/cluster-api-provider-openstack
   nestif:

Makefile

@@ -96,27 +96,34 @@ PULL_POLICY ?= Always
 LDFLAGS := $(shell source ./hack/version.sh; version::ldflags)
 
 
-
 ## --------------------------------------
 ## Testing
 ## --------------------------------------
 
+E2E_ARGS ?=
+
 $(ARTIFACTS):
 	mkdir -p $@
 
 .PHONY: test
 test: ## Run tests
 	go test -v ./...
 
+# Can be run manually, e.g. via:
+# export OPENSTACK_CLOUD_YAML_FILE="$(pwd)/clouds.yaml"
+# E2E_GINKGO_ARGS="-stream -focus='default'" E2E_ARGS="-use-existing-cluster='true'" make test-e2e
+E2E_GINKGO_ARGS ?= -stream
+.PHONY: test-e2e ## Run e2e tests using clusterctl
+test-e2e: $(GINKGO) $(KIND) $(KUSTOMIZE) e2e-image ## Run e2e tests
+	time $(GINKGO) -trace -progress -v -tags=e2e --nodes=2 $(E2E_GINKGO_ARGS) ./test/e2e/suites/e2e/... -- -config-path="$(E2E_CONF_PATH)" -artifacts-folder="$(ARTIFACTS)" --data-folder="$(E2E_DATA_DIR)" $(E2E_ARGS)
+
 .PHONY: e2e-image
 e2e-image: docker-pull-prerequisites
 	docker build -f Dockerfile --tag="gcr.io/k8s-staging-capi-openstack/capi-openstack-controller-amd64:e2e" .
 
-E2E_ARGS ?=
 CONFORMANCE_E2E_ARGS ?= -kubetest.config-file=$(KUBETEST_CONF_PATH)
 CONFORMANCE_E2E_ARGS += $(E2E_ARGS)
 CONFORMANCE_GINKGO_ARGS ?= -stream
-CONFORMANCE_GINKGO_ARGS += $(GINKGO_ARGS)
 .PHONY: test-conformance
 test-conformance: $(GINKGO) $(KIND) $(KUSTOMIZE) e2e-image ## Run clusterctl based conformance test on workload cluster (requires Docker).
 	time $(GINKGO) -trace -progress -v -tags=e2e -focus="conformance" $(CONFORMANCE_GINKGO_ARGS) ./test/e2e/suites/conformance/... -- -config-path="$(E2E_CONF_PATH)" -artifacts-folder="$(ARTIFACTS)" --data-folder="$(E2E_DATA_DIR)" $(CONFORMANCE_E2E_ARGS)

controllers/openstackcluster_controller.go

@@ -180,7 +180,6 @@ func reconcileDelete(ctx context.Context, log logr.Logger, client client.Client,
 		}
 	}
 
-	// Delete other things
 	if workerSecGroup := openStackCluster.Status.WorkerSecurityGroup; workerSecGroup != nil {
 		log.Info("Deleting worker security group", "name", workerSecGroup.Name)
 		if err = networkingService.DeleteSecurityGroups(workerSecGroup); err != nil {

go.mod

@@ -15,6 +15,7 @@ require (
 	k8s.io/api v0.21.0-beta.0
 	k8s.io/apimachinery v0.21.0-beta.0
 	k8s.io/client-go v0.21.0-beta.0
+	k8s.io/component-base v0.21.0-beta.0
 	k8s.io/klog/v2 v2.5.0
 	k8s.io/utils v0.0.0-20210111153108-fddb29f9d009
 	sigs.k8s.io/cluster-api v0.3.11-0.20210310224224-a9144a861bf4

hack/ci/devstack-cloud-init.yaml.tpl

@@ -1,8 +1,8 @@
 #cloud-config
-hostname: openstack
-password: ubuntu
-chpasswd: { expire: False }
-ssh_pwauth: True
+hostname: localhost
+users:
+- name: ubuntu
+  lock_passwd: true
 write_files:
   - content: |
       net.ipv4.ip_forward=1
@@ -22,8 +22,11 @@ write_files:
       # from https://raw.githubusercontent.com/openstack/octavia/master/devstack/contrib/new-octavia-devstack.sh
       git clone -b stable/${OPENSTACK_RELEASE} https://github.com/openstack/devstack.git /tmp/devstack
 
-      cat <<EOF > /tmp/devstack/localrc
+      cat <<EOF > /tmp/devstack/local.conf
+
+      [[local|localrc]]
       GIT_BASE=https://github.com
+      HOST_IP=10.0.2.15
 
       # Neutron
       enable_plugin neutron https://github.com/openstack/neutron stable/${OPENSTACK_RELEASE}
@@ -78,12 +81,26 @@ write_files:
 
       # Don't download default images, just our test images
       DOWNLOAD_DEFAULT_IMAGES=False
-      IMAGE_URLS="https://github.com/sbueringer/image-builder/releases/download/v1.18.15-01/ubuntu-2004-kube-v1.18.15.qcow2,"
-      IMAGE_URLS+="http://download.cirros-cloud.net/0.5.1/cirros-0.5.1-x86_64-disk.img"
+      # We upload the Amphora image so it doesn't have to be build
+      IMAGE_URLS="https://github.com/sbueringer/cluster-api-provider-openstack-images/releases/download/amphora-victoria-1/amphora-x64-haproxy.qcow2"
+
+      # See: https://docs.openstack.org/nova/victoria/configuration/sample-config.html
+      # Helpful commands (on the devstack VM):
+      # * openstack resource provider list
+      # * openstack resource provider inventory list 4aa55af2-d50a-4a53-b225-f6b22dd01044
+      # * openstack resource provider usage show 4aa55af2-d50a-4a53-b225-f6b22dd01044
+      # * openstack hypervisor stats show
+      # * openstack hypervisor list
+      # * openstack hypervisor show openstack
+      # A CPU allocation ratio von 32 gives us 32 vCPUs in devstack
+      # This should be enough to run multiple e2e tests at the same time
+      [[post-config|\$NOVA_CONF]]
+      [DEFAULT]
+      cpu_allocation_ratio = 32.0
       EOF
 
       # Create the stack user
-      /tmp/devstack/tools/create-stack-user.sh
+      HOST_IP=10.0.2.15 /tmp/devstack/tools/create-stack-user.sh
 
       # Move everything into place (/opt/stack is the $HOME folder of the stack user)
       mv /tmp/devstack /opt/stack/
@@ -95,6 +112,10 @@ write_files:
       # Add environment variables for auth/endpoints
       echo 'source /opt/stack/devstack/openrc admin admin' >> /opt/stack/.bashrc
 
+      # Upload the images so we don't have to upload them from prow
+      su - stack -c "source /opt/stack/devstack/openrc admin admin && /opt/stack/devstack/tools/upload_image.sh https://github.com/sbueringer/cluster-api-provider-openstack-images/releases/download/ubuntu-2004-v1.18.15-0/ubuntu-2004-kube-v1.18.15.qcow2"
+      su - stack -c "source /opt/stack/devstack/openrc admin admin && /opt/stack/devstack/tools/upload_image.sh http://download.cirros-cloud.net/0.5.1/cirros-0.5.1-x86_64-disk.img"
+
       sudo iptables -t nat -I POSTROUTING -o ens4 -s 172.24.4.0/24 -j MASQUERADE
       sudo iptables -I FORWARD -s 172.24.4.0/24 -j ACCEPT
 

hack/ci/devstack-on-gce-project-install.sh

@@ -56,7 +56,9 @@ function init_networks() {
   if [[ ${GCP_NETWORK_NAME} != "default" ]]; then
     if ! gcloud compute networks describe "${GCP_NETWORK_NAME}" --project "${GCP_PROJECT}" > /dev/null;
     then
-      gcloud compute networks create --project "$GCP_PROJECT" "${GCP_NETWORK_NAME}" --subnet-mode auto --quiet
+      gcloud compute networks create --project "$GCP_PROJECT" "${GCP_NETWORK_NAME}" --subnet-mode custom
+      gcloud compute networks subnets create "${GCP_NETWORK_NAME}" --project "$GCP_PROJECT" --network="${GCP_NETWORK_NAME}" --range="10.0.0.0/20" --region "${GCP_REGION}"
+
       gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-http --project "$GCP_PROJECT" \
         --allow tcp:80 --network "${GCP_NETWORK_NAME}" --quiet
       gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-https --project "$GCP_PROJECT" \
@@ -124,13 +126,13 @@ main() {
       --project "${GCP_PROJECT}" \
       --zone "${GCP_ZONE}" \
       --image ubuntu-2004-nested \
-      --boot-disk-size 100G \
+      --boot-disk-size 300G \
       --boot-disk-type pd-ssd \
       --can-ip-forward \
       --tags http-server,https-server,novnc,openstack-apis \
       --min-cpu-platform "${GCP_MACHINE_MIN_CPU_PLATFORM}" \
       --machine-type "${GCP_MACHINE_TYPE}" \
-      --network-interface=network="${CLUSTER_NAME}-mynetwork,subnet=${CLUSTER_NAME}-mynetwork,aliases=/24" \
+      --network-interface="private-network-ip=10.0.2.15,network=${CLUSTER_NAME}-mynetwork,subnet=${CLUSTER_NAME}-mynetwork" \
       --metadata-from-file user-data=./hack/ci/devstack-cloud-init.yaml
   fi
 
@@ -192,13 +194,18 @@ main() {
   openstack availability zone list
   openstack domain list
 
+  # the flavors are created in a way that we can execute at least 2 e2e tests in parallel (overall we have 32 vCPUs)
   openstack flavor delete m1.tiny
   openstack flavor create --ram 512 --disk 1 --vcpus 1 --public --id 1 m1.tiny --property hw_rng:allowed='True'
   openstack flavor delete m1.small
   openstack flavor create --ram 4192 --disk 10 --vcpus 2 --public --id 2 m1.small --property hw_rng:allowed='True'
   openstack flavor delete m1.medium
   openstack flavor create --ram 6144 --disk 10 --vcpus 4 --public --id 3 m1.medium --property hw_rng:allowed='True'
 
+  # Adjust the CPU quota
+  openstack quota set --cores 32 demo
+  openstack quota set --secgroups 50 demo
+
   export OS_TENANT_NAME=demo
   export OS_USERNAME=demo
   export OS_PROJECT_NAME=demo

main.go

@@ -28,6 +28,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
@@ -120,6 +121,7 @@ func InitFlags(fs *pflag.FlagSet) {
 
 func main() {
 	InitFlags(pflag.CommandLine)
+	pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
 	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
 	pflag.Parse()
 

pkg/cloud/services/compute/instance.go

@@ -234,7 +234,10 @@ func createInstance(is *Service, clusterName string, i *infrav1.Instance) (*infr
 	}
 
 	if i.Subnet != "" && accessIPv4 == "" {
-		return nil, fmt.Errorf("no ports with fixed IPs found on Subnet \"%s\"", i.Subnet)
+		if errd := deletePorts(is, portsList); errd != nil {
+			return nil, fmt.Errorf("no ports with fixed IPs found on Subnet %q: error cleaning up ports: %v", i.Subnet, errd)
+		}
+		return nil, fmt.Errorf("no ports with fixed IPs found on Subnet %q", i.Subnet)
 	}
 
 	flavorID, err := flavors.IDFromName(is.computeClient, i.Flavor)
@@ -266,7 +269,7 @@ func createInstance(is *Service, clusterName string, i *infrav1.Instance) (*infr
 	}).Extract()
 	if err != nil {
 		if errd := deletePorts(is, portsList); errd != nil {
-			return nil, fmt.Errorf("error recover creating Openstack instance: %v", errd)
+			return nil, fmt.Errorf("error recover creating Openstack instance: error cleaning up ports: %v", errd)
 		}
 		return nil, fmt.Errorf("error creating Openstack instance: %v", err)
 	}

pkg/cloud/services/compute/service.go

@@ -37,6 +37,7 @@ type Service struct {
 	logger         logr.Logger
 }
 
+// NewService returns an instance of the compute service.
 func NewService(client *gophercloud.ProviderClient, clientOpts *clientconfig.ClientOpts, logger logr.Logger) (*Service, error) {
 	identityClient, err := openstack.NewIdentityV3(client, gophercloud.EndpointOpts{
 		Region: "",
@@ -65,16 +66,21 @@ func NewService(client *gophercloud.ProviderClient, clientOpts *clientconfig.Cli
 	if err != nil {
 		return nil, fmt.Errorf("failed to create image service client: %v", err)
 	}
-	var projectID string
-	if clientOpts.AuthInfo != nil {
-		projectID = clientOpts.AuthInfo.ProjectID
-		if projectID == "" && clientOpts.AuthInfo.ProjectName != "" {
-			projectID, err = provider.GetProjectID(client, clientOpts.AuthInfo.ProjectName)
-			if err != nil {
-				return nil, fmt.Errorf("error retrieveing project id: %v", err)
-			}
+
+	if clientOpts.AuthInfo == nil {
+		return nil, fmt.Errorf("failed to get project id: authInfo must be set: %v", err)
+	}
+
+	projectID := clientOpts.AuthInfo.ProjectID
+	if projectID == "" && clientOpts.AuthInfo.ProjectName != "" {
+		projectID, err = provider.GetProjectID(client, clientOpts.AuthInfo.ProjectName)
+		if err != nil {
+			return nil, fmt.Errorf("error retrieveing project id: %v", err)
 		}
 	}
+	if projectID == "" {
+		return nil, fmt.Errorf("failed to get project id")
+	}
 
 	return &Service{
 		provider:       client,

pkg/cloud/services/networking/service.go

@@ -23,6 +23,8 @@ import (
 	"github.com/gophercloud/gophercloud"
 	"github.com/gophercloud/gophercloud/openstack"
 	"github.com/gophercloud/utils/openstack/clientconfig"
+
+	"sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/provider"
 )
 
 const (
@@ -32,8 +34,9 @@ const (
 // Service interfaces with the OpenStack Networking API.
 // It will create a network related infrastructure for the cluster, like network, subnet, router, security groups.
 type Service struct {
-	client *gophercloud.ServiceClient
-	logger logr.Logger
+	projectID string
+	client    *gophercloud.ServiceClient
+	logger    logr.Logger
 }
 
 // NewService returns an instance of the networking service.
@@ -44,8 +47,25 @@ func NewService(client *gophercloud.ProviderClient, clientOpts *clientconfig.Cli
 	if err != nil {
 		return nil, fmt.Errorf("failed to create networking service client: %v", err)
 	}
+
+	if clientOpts.AuthInfo == nil {
+		return nil, fmt.Errorf("failed to get project id: authInfo must be set: %v", err)
+	}
+
+	projectID := clientOpts.AuthInfo.ProjectID
+	if projectID == "" && clientOpts.AuthInfo.ProjectName != "" {
+		projectID, err = provider.GetProjectID(client, clientOpts.AuthInfo.ProjectName)
+		if err != nil {
+			return nil, fmt.Errorf("error retrieveing project id: %v", err)
+		}
+	}
+	if projectID == "" {
+		return nil, fmt.Errorf("failed to get project id")
+	}
+
 	return &Service{
-		client: serviceClient,
-		logger: logger,
+		projectID: projectID,
+		client:    serviceClient,
+		logger:    logger,
 	}, nil
 }

scripts/ci-e2e.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+################################################################################
+# usage: ci-conformance.sh
+#  This program runs the clusterctl conformance e2e tests.
+################################################################################
+
+set -o nounset
+set -o pipefail
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+cd "${REPO_ROOT}" || exit 1
+
+# shellcheck source=../hack/ensure-go.sh
+source "${REPO_ROOT}/hack/ensure-go.sh"
+# shellcheck source=../hack/ensure-kind.sh
+source "${REPO_ROOT}/hack/ensure-kind.sh"
+# shellcheck source=../hack/ensure-kubectl.sh
+source "${REPO_ROOT}/hack/ensure-kubectl.sh"
+
+RESOURCE_TYPE="${RESOURCE_TYPE:-"gce-project"}"
+
+ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
+mkdir -p "${ARTIFACTS}/logs/"
+
+# our exit handler (trap)
+cleanup() {
+  # stop boskos heartbeat
+  [[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}"
+
+  # will be started by the devstack installation script
+  pkill sshuttle
+}
+trap cleanup EXIT
+
+#Install requests module explicitly for HTTP calls
+python3 -m pip install requests
+
+# If BOSKOS_HOST is set then acquire a resource of type ${RESOURCE_TYPE} from Boskos.
+if [ -n "${BOSKOS_HOST:-}" ]; then
+  # Check out the account from Boskos and store the produced environment
+  # variables in a temporary file.
+  account_env_var_file="$(mktemp)"
+  python3 hack/boskos.py --get --resource-type="${RESOURCE_TYPE}" 1>"${account_env_var_file}"
+  checkout_account_status="${?}"
+
+  # If the checkout process was a success then load the account's
+  # environment variables into this process.
+  # shellcheck disable=SC1090
+  [ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}"
+
+  # Always remove the account environment variable file. It contains
+  # sensitive information.
+  rm -f "${account_env_var_file}"
+
+  if [ ! "${checkout_account_status}" = "0" ]; then
+    echo "error getting account from boskos" 1>&2
+    exit "${checkout_account_status}"
+  fi
+
+  # run the heart beat process to tell boskos that we are still
+  # using the checked out account periodically
+  python3 -u hack/boskos.py --heartbeat >> "$ARTIFACTS/logs/boskos.log" 2>&1 &
+  HEART_BEAT_PID=$!
+fi
+
+"hack/ci/devstack-on-${RESOURCE_TYPE}-install.sh"
+
+export OPENSTACK_CLOUD_YAML_FILE
+OPENSTACK_CLOUD_YAML_FILE="$(pwd)/clouds.yaml"
+make test-e2e
+test_status="${?}"
+
+# If Boskos is being used then release the resource back to Boskos.
+[ -z "${BOSKOS_HOST:-}" ] || python3 hack/boskos.py --release >> "$ARTIFACTS/logs/boskos.log" 2>&1
+
+exit "${test_status}"