diff --git a/Makefile b/Makefile index 09b2d5c333..f41bfee950 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ - #TODO add default features here -export FEATURES?=sctp +export FEATURES?=sctp performance # The environment represents the kustomize patches to apply when deploying the features export FEATURES_ENVIRONMENT?=e2e-gcp @@ -33,9 +32,9 @@ deps-update: go mod tidy && \ go mod vendor -functests: +functests: label-worker-rt feature-deploy @echo "Running Functional Tests" - FEATURES=$(FEATURES) hack/run-functests.sh + FEATURES="$(FEATURES)" hack/run-functests.sh gofmt: @echo "Running gofmt" @@ -64,4 +63,8 @@ kustomize: fi feature-deploy: kustomize - KUSTOMIZE=$(KUSTOMIZE) FEATURES_ENVIRONMENT=$(FEATURES_ENVIRONMENT) FEATURES=$(FEATURES) hack/feature-deploy.sh + KUSTOMIZE=$(KUSTOMIZE) FEATURES_ENVIRONMENT=$(FEATURES_ENVIRONMENT) FEATURES="$(FEATURES)" hack/feature-deploy.sh + +label-worker-rt: + @echo "Adding worker-rt label to one worker node" + hack/label-worker-rt.sh diff --git a/README.md b/README.md index d7db0cbd10..be72a0a08f 100644 --- a/README.md +++ b/README.md @@ -4,4 +4,17 @@ This repo contains example kustomize configs used to installed openshift feature # Contributing kustomize configs -All kustomize configs should be entirely declarative in nature. This means no bash plugin modules performing imparative tasks. Features should be installed simply by posting manifests to the cluster. After posting manifests, determining when the cluster has converged on those manifests successully should be observable. +All kustomize configs should be entirely declarative in nature. This means no bash plugin modules performing imparative tasks. Features should be installed simply by posting manifests to the cluster. After posting manifests, determining when the cluster has converged on those manifests successully should be observable. + +# Usage + +## Prerequisites + +- You need a running OCP 4.4 cluster and a valid KUBECONFIG. +- You need at least one node with the `node-role.kubernetes.io/worker-rt=""` label. + Run `make label-worker-rt` for adding it on the first `worker` node of the cluster. + +## Deployment + +- run `FEATURES_ENVIRONMENT=demo make feature-deploy`. + This will try to apply all manifests in a loop until all deployments succeeded, or until it runs into a timeout. diff --git a/feature-configs/base/performance/kustomization.yaml b/feature-configs/base/performance/kustomization.yaml new file mode 100644 index 0000000000..63ef269558 --- /dev/null +++ b/feature-configs/base/performance/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# TODO handle namespace with var + +resources: + - operator-namespace.yaml + - operator_operatorgroup.yaml + - operator_catalogsource.yaml + - operator_subscription.yaml + - performance_profile.yaml diff --git a/feature-configs/base/performance/operator-namespace.yaml b/feature-configs/base/performance/operator-namespace.yaml new file mode 100644 index 0000000000..cbde6e981a --- /dev/null +++ b/feature-configs/base/performance/operator-namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + openshift.io/cluster-monitoring: "true" + name: openshift-performance-addon +spec: {} diff --git a/feature-configs/base/performance/operator_catalogsource.yaml b/feature-configs/base/performance/operator_catalogsource.yaml new file mode 100644 index 0000000000..3399b3e269 --- /dev/null +++ b/feature-configs/base/performance/operator_catalogsource.yaml @@ -0,0 +1,13 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: performance-addon-operators-catalogsource + namespace: openshift-marketplace +spec: + displayName: Openshift Performance Addon Operators + icon: + base64data: "" + mediatype: "" + image: quay.io/openshift/performance-addon-operators-registry + publisher: Red Hat + sourceType: grpc diff --git a/feature-configs/base/performance/operator_operatorgroup.yaml b/feature-configs/base/performance/operator_operatorgroup.yaml new file mode 100644 index 0000000000..9ea9e3add1 --- /dev/null +++ b/feature-configs/base/performance/operator_operatorgroup.yaml @@ -0,0 +1,8 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-performance-addon-operatorgroup + namespace: openshift-performance-addon +spec: + targetNamespaces: + - openshift-performance-addon diff --git a/feature-configs/base/performance/operator_subscription.yaml b/feature-configs/base/performance/operator_subscription.yaml new file mode 100644 index 0000000000..f3d8288f1f --- /dev/null +++ b/feature-configs/base/performance/operator_subscription.yaml @@ -0,0 +1,10 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: performance-addon-operators-subscription + namespace: openshift-performance-addon +spec: + channel: alpha + name: performance-addon-operators + source: performance-addon-operators-catalogsource + sourceNamespace: openshift-marketplace diff --git a/feature-configs/base/performance/performance_profile.yaml b/feature-configs/base/performance/performance_profile.yaml new file mode 100644 index 0000000000..39f92e4096 --- /dev/null +++ b/feature-configs/base/performance/performance_profile.yaml @@ -0,0 +1,16 @@ +apiVersion: performance.openshift.io/v1alpha1 +kind: PerformanceProfile +metadata: + name: performance +spec: + cpu: + isolated: "1-3" + nonIsolated: "0" + reserved: "0-1" + hugepages: + defaultHugepagesSize: "1G" + pages: + - size: "1G" + count: 1 + realTimeKernel: + enabled: true diff --git a/feature-configs/demo/performance/kustomization.yaml b/feature-configs/demo/performance/kustomization.yaml new file mode 100644 index 0000000000..e864238121 --- /dev/null +++ b/feature-configs/demo/performance/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base/performance + +patchesStrategicMerge: + - operator_catalogsource.patch.yaml + - performance_profile.patch.yaml diff --git a/feature-configs/demo/performance/operator_catalogsource.patch.yaml b/feature-configs/demo/performance/operator_catalogsource.patch.yaml new file mode 100644 index 0000000000..5faa3d3055 --- /dev/null +++ b/feature-configs/demo/performance/operator_catalogsource.patch.yaml @@ -0,0 +1,7 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: performance-addon-operators-catalogsource + namespace: openshift-marketplace +spec: + image: quay.io/slintes/performance-addon-operators-registry diff --git a/feature-configs/demo/performance/performance_profile.patch.yaml b/feature-configs/demo/performance/performance_profile.patch.yaml new file mode 100644 index 0000000000..febb76d460 --- /dev/null +++ b/feature-configs/demo/performance/performance_profile.patch.yaml @@ -0,0 +1,7 @@ +apiVersion: performance.openshift.io/v1alpha1 +kind: PerformanceProfile +metadata: + name: performance +spec: + nodeSelector: + node-role.kubernetes.io/worker-rt: "" diff --git a/feature-configs/e2e-gcp/performance/kustomization.yaml b/feature-configs/e2e-gcp/performance/kustomization.yaml new file mode 100644 index 0000000000..e864238121 --- /dev/null +++ b/feature-configs/e2e-gcp/performance/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base/performance + +patchesStrategicMerge: + - operator_catalogsource.patch.yaml + - performance_profile.patch.yaml diff --git a/feature-configs/e2e-gcp/performance/operator_catalogsource.patch.yaml b/feature-configs/e2e-gcp/performance/operator_catalogsource.patch.yaml new file mode 100644 index 0000000000..5faa3d3055 --- /dev/null +++ b/feature-configs/e2e-gcp/performance/operator_catalogsource.patch.yaml @@ -0,0 +1,7 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: performance-addon-operators-catalogsource + namespace: openshift-marketplace +spec: + image: quay.io/slintes/performance-addon-operators-registry diff --git a/feature-configs/e2e-gcp/performance/performance_profile.patch.yaml b/feature-configs/e2e-gcp/performance/performance_profile.patch.yaml new file mode 100644 index 0000000000..febb76d460 --- /dev/null +++ b/feature-configs/e2e-gcp/performance/performance_profile.patch.yaml @@ -0,0 +1,7 @@ +apiVersion: performance.openshift.io/v1alpha1 +kind: PerformanceProfile +metadata: + name: performance +spec: + nodeSelector: + node-role.kubernetes.io/worker-rt: "" diff --git a/hack/feature-deploy.sh b/hack/feature-deploy.sh index 3d12d94c28..cd73d6e12b 100755 --- a/hack/feature-deploy.sh +++ b/hack/feature-deploy.sh @@ -2,11 +2,67 @@ set -e +if [ "$FEATURES_ENVIRONMENT" == "" ]; then + echo "[ERROR]: No FEATURES_ENVIRONMENT provided" + exit 1 +fi + +if [ "$FEATURES" == "" ]; then + echo "[ERROR]: No FEATURES provided" + exit 1 +fi + # expect oc to be in PATH by default -OC_TOOL="${OC_TOOL:-oc}" +export OC_TOOL="${OC_TOOL:-oc}" + +# expect kustomize to be in PATH by default +KUSTOMIZE="${KUSTOMIZE:-kustomize}" + +# Deploy features +success=0 +iterations=0 +sleep_time=10 +max_iterations=30 # results in 5 minute timeout +until [[ $success -eq 1 ]] || [[ $iterations -eq $max_iterations ]] +do + + feature_failed=0 + + for feature in $FEATURES; do + + feature_dir=feature-configs/${FEATURES_ENVIRONMENT}/${feature}/ + if [[ ! -d $feature_dir ]]; then + echo "[WARN] Feature '$feature' is not configured for environment '$FEATURES_ENVIRONMENT', skipping it" + continue + fi + + echo "[INFO] Deploying feature '$feature' for environment '$FEATURES_ENVIRONMENT'" + set +e + if ! ${KUSTOMIZE} build $feature_dir | ${OC_TOOL} apply -f - + then + echo "[WARN] Deployment of feature '$feature' failed." + feature_failed=1 + fi + set -e + + done + + if [[ $feature_failed -eq 1 ]]; then + iterations=$((iterations + 1)) + iterations_left=$((max_iterations - iterations)) + echo "[WARN] At least one deployment failed, retrying in $sleep_time sec, $iterations_left retries left" + sleep $sleep_time + continue + + fi + + # All features deployed successfully + success=1 -for f in $FEATURES; do - echo "TODO add logic to implement feature '$f' for environment '$FEATURES_ENVIRONMENT'" done -echo "ERROR: FEATURE DEPLOY SCRIPT $0 NEEDS IMPLEMENTATION" +if [[ $success -eq 1 ]]; then + echo "[INFO] Deployment successful" +else + echo "[ERROR] Deployment failed" +fi diff --git a/hack/label-worker-rt.sh b/hack/label-worker-rt.sh new file mode 100755 index 0000000000..e192eb8be0 --- /dev/null +++ b/hack/label-worker-rt.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +# expect oc to be in PATH by default +export OC_TOOL="${OC_TOOL:-oc}" + +# Label 1 worker node +echo "[INFO]: Labeling 1 worker node with worker-rt" +node=$(${OC_TOOL} get nodes --selector='node-role.kubernetes.io/worker' -o name | head -1) +${OC_TOOL} label $node node-role.kubernetes.io/worker-rt=""