diff --git a/Makefile b/Makefile index dfc0dc763..7af39a399 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,8 @@ export FEATURES?=mcp performance sctp operator-container \ registry-container \ generate-latest-dev-csv \ - test-cluster-setup - + test-cluster-setup \ + kustomize IMAGE_BUILD_CMD ?= "docker" IMAGE_REGISTRY ?= "quay.io" @@ -32,6 +32,12 @@ TARGET_GOARCH=amd64 CACHE_DIR="_cache" TOOLS_DIR="$(CACHE_DIR)/tools" +KUSTOMIZE_VERSION="v3.5.3" +KUSTOMIZE_PLATFORM ?= "linux_amd64" +KUSTOMIZE_BIN="kustomize" +KUSTOMIZE_TAR="$(KUSTOMIZE_BIN)_$(KUSTOMIZE_VERSION)_$(KUSTOMIZE_PLATFORM).tar.gz" +KUSTOMIZE="$(TOOLS_DIR)/$(KUSTOMIZE_BIN)" + OPERATOR_SDK_VERSION="v0.13.0" OPERATOR_SDK_PLATFORM ?= "x86_64-linux-gnu" OPERATOR_SDK_BIN="operator-sdk-$(OPERATOR_SDK_VERSION)-$(OPERATOR_SDK_PLATFORM)" @@ -102,9 +108,9 @@ deps-update: deploy: cluster-deploy # TODO - deprecated, will be removed soon in favor of cluster-deploy -cluster-deploy: +cluster-deploy: kustomize @echo "Deploying operator" - FULL_REGISTRY_IMAGE=$(FULL_REGISTRY_IMAGE) hack/deploy.sh + FULL_REGISTRY_IMAGE=$(FULL_REGISTRY_IMAGE) KUSTOMIZE=$(KUSTOMIZE) hack/deploy.sh cluster-clean: @echo "Deleting operator" @@ -144,4 +150,16 @@ ci-job: gofmt golint govet verify-generate build unittests test-cluster-setup: @echo "Setting up the test cluster" - hack/setup_test_cluster.sh \ No newline at end of file + hack/setup_test_cluster.sh + +kustomize: + @if [ ! -x "$(KUSTOMIZE)" ]; then\ + echo "Downloading kustomize $(KUSTOMIZE_VERSION)";\ + mkdir -p $(TOOLS_DIR);\ + curl -JL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/$(KUSTOMIZE_VERSION)/$(KUSTOMIZE_TAR) -o $(TOOLS_DIR)/$(KUSTOMIZE_TAR);\ + tar -xvf $(TOOLS_DIR)/$(KUSTOMIZE_TAR) -C $(TOOLS_DIR);\ + rm -rf $(TOOLS_DIR)/$(KUSTOMIZE_TAR);\ + chmod +x $(KUSTOMIZE);\ + else\ + echo "Using kustomize cached at $(KUSTOMIZE)";\ + fi diff --git a/cluster-setup/base/performance/kustomization.yaml b/cluster-setup/base/performance/kustomization.yaml new file mode 100644 index 000000000..0738104a1 --- /dev/null +++ b/cluster-setup/base/performance/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - performance_profile.yaml diff --git a/cluster-setup/base/performance/performance_profile.yaml b/cluster-setup/base/performance/performance_profile.yaml new file mode 100644 index 000000000..442175869 --- /dev/null +++ b/cluster-setup/base/performance/performance_profile.yaml @@ -0,0 +1,16 @@ +apiVersion: performance.openshift.io/v1alpha1 +kind: PerformanceProfile +metadata: + name: ci +spec: + cpu: + isolated: "1-3" + nonIsolated: "0" + reserved: "0-1" + hugepages: + defaultHugepagesSize: "1G" + pages: + - size: "1G" + count: 1 + realTimeKernel: + repoURL: "http://download-node-02.eng.bos.redhat.com/rhel-8/nightly/RHEL-8/latest-RHEL-8.1.1/compose/RT/x86_64/os" diff --git a/cluster-setup/ci-cluster/performance/kustomization.yaml b/cluster-setup/ci-cluster/performance/kustomization.yaml new file mode 100644 index 000000000..03ed77b5d --- /dev/null +++ b/cluster-setup/ci-cluster/performance/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +bases: + - ../../base/performance +patches: + - performance_profile.patch.yaml diff --git a/cluster-setup/ci-cluster/performance/performance_profile.patch.yaml b/cluster-setup/ci-cluster/performance/performance_profile.patch.yaml new file mode 100644 index 000000000..1f44d444a --- /dev/null +++ b/cluster-setup/ci-cluster/performance/performance_profile.patch.yaml @@ -0,0 +1,7 @@ +apiVersion: performance.openshift.io/v1alpha1 +kind: PerformanceProfile +metadata: + name: ci +spec: + nodeSelector: + node-role.kubernetes.io/worker-rt: "" diff --git a/functests/test_suite_test.go b/functests/test_suite_test.go index 33c419c40..65a43e2c8 100644 --- a/functests/test_suite_test.go +++ b/functests/test_suite_test.go @@ -4,6 +4,7 @@ package test_test import ( "flag" + "fmt" "testing" "time" @@ -13,12 +14,18 @@ import ( _ "github.com/openshift-kni/performance-addon-operators/functests/performance" // this is needed otherwise the performance test won't be executed testutils "github.com/openshift-kni/performance-addon-operators/functests/utils" testclient "github.com/openshift-kni/performance-addon-operators/functests/utils/client" + "github.com/openshift-kni/performance-addon-operators/functests/utils/machineconfigpool" "github.com/openshift-kni/performance-addon-operators/functests/utils/namespaces" + "github.com/openshift-kni/performance-addon-operators/functests/utils/nodes" + mcov1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// TODO: we should refactor tests to use client from controller-runtime package +// see - https://github.com/openshift/cluster-api-actuator-pkg/blob/master/pkg/e2e/framework/framework.go + var junitPath *string func init() { @@ -36,6 +43,7 @@ func TestTest(t *testing.T) { } var _ = BeforeSuite(func() { + // create test namespace ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: testutils.NamespaceTesting, @@ -43,6 +51,32 @@ var _ = BeforeSuite(func() { } _, err := testclient.Client.Namespaces().Create(ns) Expect(err).ToNot(HaveOccurred()) + + // lable one of workers nodes with worker RT label + workerNodes, err := nodes.GetByRole(testclient.Client, testutils.RoleWorker) + Expect(err).ToNot(HaveOccurred()) + Expect(workerNodes).ToNot(BeEmpty()) + + worker := &workerNodes[0] + workerRTLabel := fmt.Sprintf("%s/%s", testutils.LabelRole, testutils.RoleWorkerRT) + worker.Labels[workerRTLabel] = "" + _, err = testclient.Client.Nodes().Update(worker) + Expect(err).ToNot(HaveOccurred()) + + // wait for all machine config pools to finish update + mcps, err := testclient.Client.MachineConfigPools().List(metav1.ListOptions{}) + Expect(err).ToNot(HaveOccurred()) + + for _, mcp := range mcps.Items { + err := machineconfigpool.WaitForCondition( + testclient.Client, + &mcp, + mcov1.MachineConfigPoolUpdated, + corev1.ConditionTrue, + 25*time.Minute, + ) + Expect(err).ToNot(HaveOccurred()) + } }) var _ = AfterSuite(func() { diff --git a/functests/utils/consts.go b/functests/utils/consts.go index 0874b1403..6977c3e7b 100644 --- a/functests/utils/consts.go +++ b/functests/utils/consts.go @@ -10,6 +10,8 @@ const ( ) const ( + // RoleWorker contains the worker role + RoleWorker = "worker" // RoleWorkerRT contains the worker-rt role RoleWorkerRT = "worker-rt" ) diff --git a/functests/utils/machineconfigpool/machineconfigpool.go b/functests/utils/machineconfigpool/machineconfigpool.go new file mode 100644 index 000000000..8e284ac02 --- /dev/null +++ b/functests/utils/machineconfigpool/machineconfigpool.go @@ -0,0 +1,35 @@ +package machineconfigpool + +import ( + "time" + + testclient "github.com/openshift-kni/performance-addon-operators/functests/utils/client" + mcov1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +// WaitForCondition waits until the machine config pool will have specified condition type with the expected status +func WaitForCondition( + cs *testclient.ClientSet, + mcp *mcov1.MachineConfigPool, + conditionType mcov1.MachineConfigPoolConditionType, + conditionStatus corev1.ConditionStatus, + timeout time.Duration, +) error { + return wait.PollImmediate(10*time.Second, timeout, func() (bool, error) { + mcpUpdated, err := cs.MachineConfigPools().Get(mcp.Name, metav1.GetOptions{}) + if err != nil { + return false, nil + } + + for _, c := range mcpUpdated.Status.Conditions { + if c.Type == conditionType && c.Status == conditionStatus { + return true, nil + } + } + return false, nil + }) +} diff --git a/hack/clean-deploy.sh b/hack/clean-deploy.sh index e32b0eb54..49282a2fe 100755 --- a/hack/clean-deploy.sh +++ b/hack/clean-deploy.sh @@ -3,6 +3,8 @@ # expect oc to be in PATH by default OC_TOOL="${OC_TOOL:-oc}" +$OC_TOOL delete performanceprofile --all + $OC_TOOL delete ns openshift-performance-addon $OC_TOOL delete -f - <