Deploy performance components under CI

Makefile

@@ -18,8 +18,8 @@ export FEATURES?=mcp performance sctp
 	operator-container \
 	registry-container \
 	generate-latest-dev-csv \
-	test-cluster-setup
-
+	test-cluster-setup \
+	kustomize
 
 IMAGE_BUILD_CMD ?= "docker"
 IMAGE_REGISTRY ?= "quay.io"
@@ -32,6 +32,12 @@ TARGET_GOARCH=amd64
 CACHE_DIR="_cache"
 TOOLS_DIR="$(CACHE_DIR)/tools"
 
+KUSTOMIZE_VERSION="v3.5.3"
+KUSTOMIZE_PLATFORM ?= "linux_amd64"
+KUSTOMIZE_BIN="kustomize"
+KUSTOMIZE_TAR="$(KUSTOMIZE_BIN)_$(KUSTOMIZE_VERSION)_$(KUSTOMIZE_PLATFORM).tar.gz"
+KUSTOMIZE="$(TOOLS_DIR)/$(KUSTOMIZE_BIN)"
+
 OPERATOR_SDK_VERSION="v0.13.0"
 OPERATOR_SDK_PLATFORM ?= "x86_64-linux-gnu"
 OPERATOR_SDK_BIN="operator-sdk-$(OPERATOR_SDK_VERSION)-$(OPERATOR_SDK_PLATFORM)"
@@ -102,9 +108,9 @@ deps-update:
 deploy: cluster-deploy
 	# TODO - deprecated, will be removed soon in favor of cluster-deploy
 
-cluster-deploy:
+cluster-deploy: kustomize
 	@echo "Deploying operator"
-	FULL_REGISTRY_IMAGE=$(FULL_REGISTRY_IMAGE) hack/deploy.sh
+	FULL_REGISTRY_IMAGE=$(FULL_REGISTRY_IMAGE) KUSTOMIZE=$(KUSTOMIZE) hack/deploy.sh
 
 cluster-clean:
 	@echo "Deleting operator"
@@ -144,4 +150,16 @@ ci-job: gofmt golint govet verify-generate build unittests
 
 test-cluster-setup:
 	@echo "Setting up the test cluster"
-	hack/setup_test_cluster.sh
+	hack/setup_test_cluster.sh
+
+kustomize:
+	@if [ ! -x "$(KUSTOMIZE)" ]; then\
+		echo "Downloading kustomize $(KUSTOMIZE_VERSION)";\
+		mkdir -p $(TOOLS_DIR);\
+		curl -JL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/$(KUSTOMIZE_VERSION)/$(KUSTOMIZE_TAR) -o $(TOOLS_DIR)/$(KUSTOMIZE_TAR);\
+		tar -xvf $(TOOLS_DIR)/$(KUSTOMIZE_TAR) -C $(TOOLS_DIR);\
+		rm -rf $(TOOLS_DIR)/$(KUSTOMIZE_TAR);\
+		chmod +x $(KUSTOMIZE);\
+	else\
+		echo "Using kustomize cached at $(KUSTOMIZE)";\
+	fi

cluster-setup/base/performance/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - performance_profile.yaml

cluster-setup/base/performance/performance_profile.yaml

@@ -0,0 +1,16 @@
+apiVersion: performance.openshift.io/v1alpha1
+kind: PerformanceProfile
+metadata:
+  name: ci
+spec:
+  cpu:
+    isolated: "1-3"
+    nonIsolated: "0"
+    reserved: "0-1"
+  hugepages:
+    defaultHugepagesSize: "1G"
+    pages:
+    - size: "1G"
+      count: 1
+  realTimeKernel:
+    repoURL: "http://download-node-02.eng.bos.redhat.com/rhel-8/nightly/RHEL-8/latest-RHEL-8.1.1/compose/RT/x86_64/os"

cluster-setup/ci-cluster/performance/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+  - ../../base/performance
+patches:
+  - performance_profile.patch.yaml

cluster-setup/ci-cluster/performance/performance_profile.patch.yaml

@@ -0,0 +1,7 @@
+apiVersion: performance.openshift.io/v1alpha1
+kind: PerformanceProfile
+metadata:
+  name: ci
+spec:
+  nodeSelector:
+    node-role.kubernetes.io/worker-rt: ""

functests/test_suite_test.go

@@ -4,6 +4,7 @@ package test_test
 
 import (
 	"flag"
+	"fmt"
 	"testing"
 	"time"
 
@@ -13,12 +14,18 @@ import (
 	_ "github.com/openshift-kni/performance-addon-operators/functests/performance" // this is needed otherwise the performance test won't be executed
 	testutils "github.com/openshift-kni/performance-addon-operators/functests/utils"
 	testclient "github.com/openshift-kni/performance-addon-operators/functests/utils/client"
+	"github.com/openshift-kni/performance-addon-operators/functests/utils/machineconfigpool"
 	"github.com/openshift-kni/performance-addon-operators/functests/utils/namespaces"
+	"github.com/openshift-kni/performance-addon-operators/functests/utils/nodes"
+	mcov1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+// TODO: we should refactor tests to use client from controller-runtime package
+// see - https://github.com/openshift/cluster-api-actuator-pkg/blob/master/pkg/e2e/framework/framework.go
+
 var junitPath *string
 
 func init() {
@@ -36,13 +43,40 @@ func TestTest(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
+	// create test namespace
 	ns := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: testutils.NamespaceTesting,
 		},
 	}
 	_, err := testclient.Client.Namespaces().Create(ns)
 	Expect(err).ToNot(HaveOccurred())
+
+	// lable one of workers nodes with worker RT label
+	workerNodes, err := nodes.GetByRole(testclient.Client, testutils.RoleWorker)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(workerNodes).ToNot(BeEmpty())
+
+	worker := &workerNodes[0]
+	workerRTLabel := fmt.Sprintf("%s/%s", testutils.LabelRole, testutils.RoleWorkerRT)
+	worker.Labels[workerRTLabel] = ""
+	_, err = testclient.Client.Nodes().Update(worker)
+	Expect(err).ToNot(HaveOccurred())
+
+	// wait for all machine config pools to finish update
+	mcps, err := testclient.Client.MachineConfigPools().List(metav1.ListOptions{})
+	Expect(err).ToNot(HaveOccurred())
+
+	for _, mcp := range mcps.Items {
+		err := machineconfigpool.WaitForCondition(
+			testclient.Client,
+			&mcp,
+			mcov1.MachineConfigPoolUpdated,
+			corev1.ConditionTrue,
+			25*time.Minute,
+		)
+		Expect(err).ToNot(HaveOccurred())
+	}
 })
 
 var _ = AfterSuite(func() {

functests/utils/consts.go

@@ -10,6 +10,8 @@ const (
 )
 
 const (
+	// RoleWorker contains the worker role
+	RoleWorker = "worker"
 	// RoleWorkerRT contains the worker-rt role
 	RoleWorkerRT = "worker-rt"
 )

functests/utils/machineconfigpool/machineconfigpool.go

@@ -0,0 +1,35 @@
+package machineconfigpool
+
+import (
+	"time"
+
+	testclient "github.com/openshift-kni/performance-addon-operators/functests/utils/client"
+	mcov1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+// WaitForCondition waits until the machine config pool will have specified condition type with the expected status
+func WaitForCondition(
+	cs *testclient.ClientSet,
+	mcp *mcov1.MachineConfigPool,
+	conditionType mcov1.MachineConfigPoolConditionType,
+	conditionStatus corev1.ConditionStatus,
+	timeout time.Duration,
+) error {
+	return wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
+		mcpUpdated, err := cs.MachineConfigPools().Get(mcp.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, nil
+		}
+
+		for _, c := range mcpUpdated.Status.Conditions {
+			if c.Type == conditionType && c.Status == conditionStatus {
+				return true, nil
+			}
+		}
+		return false, nil
+	})
+}

hack/clean-deploy.sh

@@ -3,6 +3,8 @@
 # expect oc to be in PATH by default
 OC_TOOL="${OC_TOOL:-oc}"
 
+$OC_TOOL delete performanceprofile --all
+
 $OC_TOOL delete ns openshift-performance-addon
 
 $OC_TOOL delete -f - <<EOF

hack/deploy.sh

@@ -6,12 +6,12 @@ set -e
 OC_TOOL="${OC_TOOL:-oc}"
 
 # Override the image name when this is invoked from openshift ci                               
-if [ -n "$OPENSHIFT_BUILD_NAMESPACE" ]; then                                                   
+if [ -n "${OPENSHIFT_BUILD_NAMESPACE}" ]; then                                                   
         FULL_REGISTRY_IMAGE="registry.svc.ci.openshift.org/${OPENSHIFT_BUILD_NAMESPACE}/stable:performance-addon-operators-registry"
         echo "Openshift CI detected, deploying using image $FULL_REGISTRY_IMAGE"                   
 fi   
 
-$OC_TOOL apply -f - <<EOF
+${OC_TOOL} apply -f - <<EOF
 ---
 apiVersion: v1
 kind: Namespace
@@ -22,7 +22,7 @@ metadata:
 spec: {}
 EOF
 
-$OC_TOOL apply -f - <<EOF
+${OC_TOOL} apply -f - <<EOF
 ---
 apiVersion: operators.coreos.com/v1
 kind: OperatorGroup
@@ -34,7 +34,7 @@ spec:
   - openshift-performance-addon
 EOF
 
-$OC_TOOL apply -f - <<EOF
+${OC_TOOL} apply -f - <<EOF
 ---
 apiVersion: operators.coreos.com/v1alpha1
 kind: CatalogSource
@@ -51,7 +51,7 @@ spec:
   sourceType: grpc
 EOF
 
-$OC_TOOL apply -f - <<EOF
+${OC_TOOL} apply -f - <<EOF
 apiVersion: operators.coreos.com/v1alpha1
 kind: Subscription
 metadata:
@@ -64,3 +64,12 @@ spec:
   sourceNamespace: openshift-marketplace
 EOF
 
+# Wait for performance-addon-operator deployment to be ready
+until ${OC_TOOL} -n openshift-performance-addon get deploy/performance-operator; do
+    echo "[INFO]: get performance-operator deployment"
+    sleep 10
+done
+${OC_TOOL} -n openshift-performance-addon wait deploy/performance-operator --for condition=Available --timeout 5m
+
+# Deploy performance-profile custom resource
+${KUSTOMIZE} build cluster-setup/ci-cluster/performance/ | envsubst | ${OC_TOOL} apply -f -

openshift-ci/Dockerfile.deploy

@@ -3,6 +3,7 @@ LABEL com.redhat.delivery.appregistry=true
 
 ENV LANG=en_US.utf8
 
+COPY build/assets /assets
 COPY performance-addon-operators /usr/local/bin/performance-operator
 USER 1001