From 7e013c994780a923953b94ce0479bfb90069ae71 Mon Sep 17 00:00:00 2001 From: Marc Sluiter Date: Fri, 10 Jan 2020 15:07:24 +0100 Subject: [PATCH 1/4] Added kustomization for performance Signed-off-by: Marc Sluiter --- .gitignore | 2 + Makefile | 4 +- .../base/namespace/kustomization.yaml | 5 ++ feature-configs/base/namespace/namespace.yaml | 7 +++ .../performance-operator/kustomization.yaml | 7 +++ .../operator_catalogsource.yaml | 13 ++++++ .../operator_operatorgroup.yaml | 8 ++++ .../operator_subscription.yaml | 10 ++++ .../performance-profile/kustomization.yaml | 5 ++ .../performance_profile.yaml | 17 +++++++ .../performance-operator/kustomization.yaml | 10 ++++ .../operator_catalogsource.patch.yaml | 7 +++ .../demo/performance-operator/wait_for_it.sh | 8 ++++ .../performance-profile/kustomization.yaml | 9 ++++ .../performance_profile.patch.yaml | 8 ++++ .../performance-operator/kustomization.yaml | 12 +++++ .../operator_catalogsource.patch.yaml.in | 7 +++ .../performance-operator/wait_for_it.sh | 8 ++++ .../performance-profile/kustomization.yaml | 9 ++++ .../performance_profile.patch.yaml | 8 ++++ hack/feature-deploy.sh | 46 +++++++++++++++++-- 21 files changed, 203 insertions(+), 7 deletions(-) create mode 100644 feature-configs/base/namespace/kustomization.yaml create mode 100644 feature-configs/base/namespace/namespace.yaml create mode 100644 feature-configs/base/performance-operator/kustomization.yaml create mode 100644 feature-configs/base/performance-operator/operator_catalogsource.yaml create mode 100644 feature-configs/base/performance-operator/operator_operatorgroup.yaml create mode 100644 feature-configs/base/performance-operator/operator_subscription.yaml create mode 100644 feature-configs/base/performance-profile/kustomization.yaml create mode 100644 feature-configs/base/performance-profile/performance_profile.yaml create mode 100644 feature-configs/demo/performance-operator/kustomization.yaml create mode 100644 feature-configs/demo/performance-operator/operator_catalogsource.patch.yaml create mode 100755 feature-configs/demo/performance-operator/wait_for_it.sh create mode 100644 feature-configs/demo/performance-profile/kustomization.yaml create mode 100644 feature-configs/demo/performance-profile/performance_profile.patch.yaml create mode 100644 feature-configs/e2e-gcp/performance-operator/kustomization.yaml create mode 100644 feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml.in create mode 100755 feature-configs/e2e-gcp/performance-operator/wait_for_it.sh create mode 100644 feature-configs/e2e-gcp/performance-profile/kustomization.yaml create mode 100644 feature-configs/e2e-gcp/performance-profile/performance_profile.patch.yaml diff --git a/.gitignore b/.gitignore index c0ec4df66c..96f75ec2f4 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,5 @@ tags .vscode/* .history # End of https://www.gitignore.io/api/go,vim,emacs,visualstudiocode + +/feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml diff --git a/Makefile b/Makefile index 09b2d5c333..2680aea5b8 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ #TODO add default features here -export FEATURES?=sctp +export FEATURES?=sctp performance-operator performance-profile # The environment represents the kustomize patches to apply when deploying the features export FEATURES_ENVIRONMENT?=e2e-gcp @@ -64,4 +64,4 @@ kustomize: fi feature-deploy: kustomize - KUSTOMIZE=$(KUSTOMIZE) FEATURES_ENVIRONMENT=$(FEATURES_ENVIRONMENT) FEATURES=$(FEATURES) hack/feature-deploy.sh + KUSTOMIZE=$(KUSTOMIZE) FEATURES_ENVIRONMENT=$(FEATURES_ENVIRONMENT) FEATURES="$(FEATURES)" hack/feature-deploy.sh diff --git a/feature-configs/base/namespace/kustomization.yaml b/feature-configs/base/namespace/kustomization.yaml new file mode 100644 index 0000000000..dca4a51888 --- /dev/null +++ b/feature-configs/base/namespace/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml diff --git a/feature-configs/base/namespace/namespace.yaml b/feature-configs/base/namespace/namespace.yaml new file mode 100644 index 0000000000..26cec21269 --- /dev/null +++ b/feature-configs/base/namespace/namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + openshift.io/cluster-monitoring: "true" + name: openshift-performance-addon +spec: {} \ No newline at end of file diff --git a/feature-configs/base/performance-operator/kustomization.yaml b/feature-configs/base/performance-operator/kustomization.yaml new file mode 100644 index 0000000000..ce781793df --- /dev/null +++ b/feature-configs/base/performance-operator/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - operator_operatorgroup.yaml + - operator_catalogsource.yaml + - operator_subscription.yaml diff --git a/feature-configs/base/performance-operator/operator_catalogsource.yaml b/feature-configs/base/performance-operator/operator_catalogsource.yaml new file mode 100644 index 0000000000..3399b3e269 --- /dev/null +++ b/feature-configs/base/performance-operator/operator_catalogsource.yaml @@ -0,0 +1,13 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: performance-addon-operators-catalogsource + namespace: openshift-marketplace +spec: + displayName: Openshift Performance Addon Operators + icon: + base64data: "" + mediatype: "" + image: quay.io/openshift/performance-addon-operators-registry + publisher: Red Hat + sourceType: grpc diff --git a/feature-configs/base/performance-operator/operator_operatorgroup.yaml b/feature-configs/base/performance-operator/operator_operatorgroup.yaml new file mode 100644 index 0000000000..9ea9e3add1 --- /dev/null +++ b/feature-configs/base/performance-operator/operator_operatorgroup.yaml @@ -0,0 +1,8 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-performance-addon-operatorgroup + namespace: openshift-performance-addon +spec: + targetNamespaces: + - openshift-performance-addon diff --git a/feature-configs/base/performance-operator/operator_subscription.yaml b/feature-configs/base/performance-operator/operator_subscription.yaml new file mode 100644 index 0000000000..f3d8288f1f --- /dev/null +++ b/feature-configs/base/performance-operator/operator_subscription.yaml @@ -0,0 +1,10 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: performance-addon-operators-subscription + namespace: openshift-performance-addon +spec: + channel: alpha + name: performance-addon-operators + source: performance-addon-operators-catalogsource + sourceNamespace: openshift-marketplace diff --git a/feature-configs/base/performance-profile/kustomization.yaml b/feature-configs/base/performance-profile/kustomization.yaml new file mode 100644 index 0000000000..0738104a1f --- /dev/null +++ b/feature-configs/base/performance-profile/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - performance_profile.yaml diff --git a/feature-configs/base/performance-profile/performance_profile.yaml b/feature-configs/base/performance-profile/performance_profile.yaml new file mode 100644 index 0000000000..3dc9b36563 --- /dev/null +++ b/feature-configs/base/performance-profile/performance_profile.yaml @@ -0,0 +1,17 @@ +apiVersion: performance.openshift.io/v1alpha1 +kind: PerformanceProfile +metadata: + name: performance + namespace: openshift-performance-addon +spec: + cpu: + isolated: "1-3" + nonIsolated: "0" + reserved: "0-1" + hugepages: + defaultHugepagesSize: "1G" + pages: + - size: "1G" + count: 1 + realTimeKernel: + enabled: true diff --git a/feature-configs/demo/performance-operator/kustomization.yaml b/feature-configs/demo/performance-operator/kustomization.yaml new file mode 100644 index 0000000000..6ee89957b1 --- /dev/null +++ b/feature-configs/demo/performance-operator/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# TODO handle namespace with var + +resources: + - ../../base/namespace + - ../../base/performance-operator +patchesStrategicMerge: + - operator_catalogsource.patch.yaml diff --git a/feature-configs/demo/performance-operator/operator_catalogsource.patch.yaml b/feature-configs/demo/performance-operator/operator_catalogsource.patch.yaml new file mode 100644 index 0000000000..5faa3d3055 --- /dev/null +++ b/feature-configs/demo/performance-operator/operator_catalogsource.patch.yaml @@ -0,0 +1,7 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: performance-addon-operators-catalogsource + namespace: openshift-marketplace +spec: + image: quay.io/slintes/performance-addon-operators-registry diff --git a/feature-configs/demo/performance-operator/wait_for_it.sh b/feature-configs/demo/performance-operator/wait_for_it.sh new file mode 100755 index 0000000000..90408b6d84 --- /dev/null +++ b/feature-configs/demo/performance-operator/wait_for_it.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Wait for performance-addon-operator deployment to be ready +until ${OC_TOOL} -n openshift-performance-addon get deploy/performance-operator; do + echo "[INFO]: get performance-operator deployment" + sleep 10 +done +${OC_TOOL} -n openshift-performance-addon wait deploy/performance-operator --for condition=Available --timeout 5m diff --git a/feature-configs/demo/performance-profile/kustomization.yaml b/feature-configs/demo/performance-profile/kustomization.yaml new file mode 100644 index 0000000000..7beaf6a442 --- /dev/null +++ b/feature-configs/demo/performance-profile/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# TODO handle namespace with var + +resources: + - ../../base/performance-profile +patchesStrategicMerge: + - performance_profile.patch.yaml diff --git a/feature-configs/demo/performance-profile/performance_profile.patch.yaml b/feature-configs/demo/performance-profile/performance_profile.patch.yaml new file mode 100644 index 0000000000..a359742f0d --- /dev/null +++ b/feature-configs/demo/performance-profile/performance_profile.patch.yaml @@ -0,0 +1,8 @@ +apiVersion: performance.openshift.io/v1alpha1 +kind: PerformanceProfile +metadata: + name: performance + namespace: openshift-performance-addon +spec: + nodeSelector: + node-role.kubernetes.io/worker-rt: "" diff --git a/feature-configs/e2e-gcp/performance-operator/kustomization.yaml b/feature-configs/e2e-gcp/performance-operator/kustomization.yaml new file mode 100644 index 0000000000..bcd9669036 --- /dev/null +++ b/feature-configs/e2e-gcp/performance-operator/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# TODO handle namespace with var + +resources: + - ../../base/namespace + - ../../base/performance-operator + +patchesStrategicMerge: + - operator_catalogsource.patch.yaml + diff --git a/feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml.in b/feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml.in new file mode 100644 index 0000000000..021e3c1ec7 --- /dev/null +++ b/feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml.in @@ -0,0 +1,7 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: performance-addon-operators-catalogsource + namespace: openshift-marketplace +spec: + # will added by hack/feature-deploy.sh diff --git a/feature-configs/e2e-gcp/performance-operator/wait_for_it.sh b/feature-configs/e2e-gcp/performance-operator/wait_for_it.sh new file mode 100755 index 0000000000..90408b6d84 --- /dev/null +++ b/feature-configs/e2e-gcp/performance-operator/wait_for_it.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Wait for performance-addon-operator deployment to be ready +until ${OC_TOOL} -n openshift-performance-addon get deploy/performance-operator; do + echo "[INFO]: get performance-operator deployment" + sleep 10 +done +${OC_TOOL} -n openshift-performance-addon wait deploy/performance-operator --for condition=Available --timeout 5m diff --git a/feature-configs/e2e-gcp/performance-profile/kustomization.yaml b/feature-configs/e2e-gcp/performance-profile/kustomization.yaml new file mode 100644 index 0000000000..7beaf6a442 --- /dev/null +++ b/feature-configs/e2e-gcp/performance-profile/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# TODO handle namespace with var + +resources: + - ../../base/performance-profile +patchesStrategicMerge: + - performance_profile.patch.yaml diff --git a/feature-configs/e2e-gcp/performance-profile/performance_profile.patch.yaml b/feature-configs/e2e-gcp/performance-profile/performance_profile.patch.yaml new file mode 100644 index 0000000000..a359742f0d --- /dev/null +++ b/feature-configs/e2e-gcp/performance-profile/performance_profile.patch.yaml @@ -0,0 +1,8 @@ +apiVersion: performance.openshift.io/v1alpha1 +kind: PerformanceProfile +metadata: + name: performance + namespace: openshift-performance-addon +spec: + nodeSelector: + node-role.kubernetes.io/worker-rt: "" diff --git a/hack/feature-deploy.sh b/hack/feature-deploy.sh index 3d12d94c28..4fd9efdafb 100755 --- a/hack/feature-deploy.sh +++ b/hack/feature-deploy.sh @@ -2,11 +2,47 @@ set -e +if [ "$FEATURES_ENVIRONMENT" == "" ]; then + echo "[ERROR]: No FEATURES_ENVIRONMENT provided" + exit 1 +fi + +if [ "$FEATURES" == "" ]; then + echo "[ERROR]: No FEATURES provided" + exit 1 +fi + # expect oc to be in PATH by default -OC_TOOL="${OC_TOOL:-oc}" +export OC_TOOL="${OC_TOOL:-oc}" -for f in $FEATURES; do - echo "TODO add logic to implement feature '$f' for environment '$FEATURES_ENVIRONMENT'" -done +# expect kustomize to be in PATH by default +KUSTOMIZE="${KUSTOMIZE:-kustomize}" + +# Label 1 worker node +echo "[INFO]:labeling 1 worker node with worker-rt" +node=$(${OC_TOOL} get nodes --selector='node-role.kubernetes.io/worker' -o name | head -1) +${OC_TOOL} label --overwrite=true $node node-role.kubernetes.io/worker-rt="" + +# Override the image name when this is invoked from openshift ci +# Not ideal, but kustomize does not support env vars directly :/ +if [ -n "${OPENSHIFT_BUILD_NAMESPACE}" ]; then + echo "[INFO]: Openshift CI detected, deploying using image $FULL_REGISTRY_IMAGE" + FULL_REGISTRY_IMAGE="registry.svc.ci.openshift.org/${OPENSHIFT_BUILD_NAMESPACE}/stable:performance-addon-operators-registry" + cp feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml.in feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml + echo " $FULL_REGISTRY_IMAGE" >> feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml +fi -echo "ERROR: FEATURE DEPLOY SCRIPT $0 NEEDS IMPLEMENTATION" +# Deploy features +for feature in $FEATURES; do + + echo "[INFO]: Deploying feature '$feature' for environment '$FEATURES_ENVIRONMENT'" + ${KUSTOMIZE} build feature-configs/${FEATURES_ENVIRONMENT}/${feature}/ | ${OC_TOOL} apply -f - + + # Wait for feature + if [ -f "feature-configs/${FEATURES_ENVIRONMENT}/${feature}/wait_for_it.sh" ]; then + echo "[INFO]: waiting for $feature to be deployed" + feature-configs/${FEATURES_ENVIRONMENT}/${feature}/wait_for_it.sh + echo "[INFO]: $feature was deployed" + fi; + +done From 335f4cc5a4dc82c32c620245e813a590006ab514 Mon Sep 17 00:00:00 2001 From: Marc Sluiter Date: Mon, 13 Jan 2020 12:44:45 +0100 Subject: [PATCH 2/4] Moved namespace to operator, removed ${OPENSHIFT_BUILD_NAMESPACE} handling (we don't build here at all) Signed-off-by: Marc Sluiter --- .gitignore | 2 -- feature-configs/base/namespace/kustomization.yaml | 5 ----- .../base/performance-operator/kustomization.yaml | 3 +++ .../operator-namespace.yaml} | 0 .../base/performance-profile/performance_profile.yaml | 1 - .../demo/performance-operator/kustomization.yaml | 4 +--- .../demo/performance-profile/kustomization.yaml | 3 +-- .../performance-profile/performance_profile.patch.yaml | 1 - .../e2e-gcp/performance-operator/kustomization.yaml | 3 --- ...e.patch.yaml.in => operator_catalogsource.patch.yaml} | 2 +- .../e2e-gcp/performance-profile/kustomization.yaml | 3 +-- .../performance-profile/performance_profile.patch.yaml | 1 - hack/feature-deploy.sh | 9 --------- 13 files changed, 7 insertions(+), 30 deletions(-) delete mode 100644 feature-configs/base/namespace/kustomization.yaml rename feature-configs/base/{namespace/namespace.yaml => performance-operator/operator-namespace.yaml} (100%) rename feature-configs/e2e-gcp/performance-operator/{operator_catalogsource.patch.yaml.in => operator_catalogsource.patch.yaml} (72%) diff --git a/.gitignore b/.gitignore index 96f75ec2f4..c0ec4df66c 100644 --- a/.gitignore +++ b/.gitignore @@ -74,5 +74,3 @@ tags .vscode/* .history # End of https://www.gitignore.io/api/go,vim,emacs,visualstudiocode - -/feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml diff --git a/feature-configs/base/namespace/kustomization.yaml b/feature-configs/base/namespace/kustomization.yaml deleted file mode 100644 index dca4a51888..0000000000 --- a/feature-configs/base/namespace/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - namespace.yaml diff --git a/feature-configs/base/performance-operator/kustomization.yaml b/feature-configs/base/performance-operator/kustomization.yaml index ce781793df..6a5c4f281b 100644 --- a/feature-configs/base/performance-operator/kustomization.yaml +++ b/feature-configs/base/performance-operator/kustomization.yaml @@ -1,7 +1,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +# TODO handle namespace with var + resources: + - operator-namespace.yaml - operator_operatorgroup.yaml - operator_catalogsource.yaml - operator_subscription.yaml diff --git a/feature-configs/base/namespace/namespace.yaml b/feature-configs/base/performance-operator/operator-namespace.yaml similarity index 100% rename from feature-configs/base/namespace/namespace.yaml rename to feature-configs/base/performance-operator/operator-namespace.yaml diff --git a/feature-configs/base/performance-profile/performance_profile.yaml b/feature-configs/base/performance-profile/performance_profile.yaml index 3dc9b36563..39f92e4096 100644 --- a/feature-configs/base/performance-profile/performance_profile.yaml +++ b/feature-configs/base/performance-profile/performance_profile.yaml @@ -2,7 +2,6 @@ apiVersion: performance.openshift.io/v1alpha1 kind: PerformanceProfile metadata: name: performance - namespace: openshift-performance-addon spec: cpu: isolated: "1-3" diff --git a/feature-configs/demo/performance-operator/kustomization.yaml b/feature-configs/demo/performance-operator/kustomization.yaml index 6ee89957b1..e754a456f9 100644 --- a/feature-configs/demo/performance-operator/kustomization.yaml +++ b/feature-configs/demo/performance-operator/kustomization.yaml @@ -1,10 +1,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -# TODO handle namespace with var - resources: - - ../../base/namespace - ../../base/performance-operator + patchesStrategicMerge: - operator_catalogsource.patch.yaml diff --git a/feature-configs/demo/performance-profile/kustomization.yaml b/feature-configs/demo/performance-profile/kustomization.yaml index 7beaf6a442..bd883f3ada 100644 --- a/feature-configs/demo/performance-profile/kustomization.yaml +++ b/feature-configs/demo/performance-profile/kustomization.yaml @@ -1,9 +1,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -# TODO handle namespace with var - resources: - ../../base/performance-profile + patchesStrategicMerge: - performance_profile.patch.yaml diff --git a/feature-configs/demo/performance-profile/performance_profile.patch.yaml b/feature-configs/demo/performance-profile/performance_profile.patch.yaml index a359742f0d..febb76d460 100644 --- a/feature-configs/demo/performance-profile/performance_profile.patch.yaml +++ b/feature-configs/demo/performance-profile/performance_profile.patch.yaml @@ -2,7 +2,6 @@ apiVersion: performance.openshift.io/v1alpha1 kind: PerformanceProfile metadata: name: performance - namespace: openshift-performance-addon spec: nodeSelector: node-role.kubernetes.io/worker-rt: "" diff --git a/feature-configs/e2e-gcp/performance-operator/kustomization.yaml b/feature-configs/e2e-gcp/performance-operator/kustomization.yaml index bcd9669036..f628977725 100644 --- a/feature-configs/e2e-gcp/performance-operator/kustomization.yaml +++ b/feature-configs/e2e-gcp/performance-operator/kustomization.yaml @@ -1,10 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -# TODO handle namespace with var - resources: - - ../../base/namespace - ../../base/performance-operator patchesStrategicMerge: diff --git a/feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml.in b/feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml similarity index 72% rename from feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml.in rename to feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml index 021e3c1ec7..5faa3d3055 100644 --- a/feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml.in +++ b/feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml @@ -4,4 +4,4 @@ metadata: name: performance-addon-operators-catalogsource namespace: openshift-marketplace spec: - # will added by hack/feature-deploy.sh + image: quay.io/slintes/performance-addon-operators-registry diff --git a/feature-configs/e2e-gcp/performance-profile/kustomization.yaml b/feature-configs/e2e-gcp/performance-profile/kustomization.yaml index 7beaf6a442..bd883f3ada 100644 --- a/feature-configs/e2e-gcp/performance-profile/kustomization.yaml +++ b/feature-configs/e2e-gcp/performance-profile/kustomization.yaml @@ -1,9 +1,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -# TODO handle namespace with var - resources: - ../../base/performance-profile + patchesStrategicMerge: - performance_profile.patch.yaml diff --git a/feature-configs/e2e-gcp/performance-profile/performance_profile.patch.yaml b/feature-configs/e2e-gcp/performance-profile/performance_profile.patch.yaml index a359742f0d..febb76d460 100644 --- a/feature-configs/e2e-gcp/performance-profile/performance_profile.patch.yaml +++ b/feature-configs/e2e-gcp/performance-profile/performance_profile.patch.yaml @@ -2,7 +2,6 @@ apiVersion: performance.openshift.io/v1alpha1 kind: PerformanceProfile metadata: name: performance - namespace: openshift-performance-addon spec: nodeSelector: node-role.kubernetes.io/worker-rt: "" diff --git a/hack/feature-deploy.sh b/hack/feature-deploy.sh index 4fd9efdafb..a0f894dce6 100755 --- a/hack/feature-deploy.sh +++ b/hack/feature-deploy.sh @@ -23,15 +23,6 @@ echo "[INFO]:labeling 1 worker node with worker-rt" node=$(${OC_TOOL} get nodes --selector='node-role.kubernetes.io/worker' -o name | head -1) ${OC_TOOL} label --overwrite=true $node node-role.kubernetes.io/worker-rt="" -# Override the image name when this is invoked from openshift ci -# Not ideal, but kustomize does not support env vars directly :/ -if [ -n "${OPENSHIFT_BUILD_NAMESPACE}" ]; then - echo "[INFO]: Openshift CI detected, deploying using image $FULL_REGISTRY_IMAGE" - FULL_REGISTRY_IMAGE="registry.svc.ci.openshift.org/${OPENSHIFT_BUILD_NAMESPACE}/stable:performance-addon-operators-registry" - cp feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml.in feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml - echo " $FULL_REGISTRY_IMAGE" >> feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml -fi - # Deploy features for feature in $FEATURES; do From c02b05290ef2bf5a54b0dad011f98e9b6b045592 Mon Sep 17 00:00:00 2001 From: Marc Sluiter Date: Mon, 13 Jan 2020 16:32:01 +0100 Subject: [PATCH 3/4] Merged performace-operator and -profile, retry instead wait Signed-off-by: Marc Sluiter --- Makefile | 7 ++- .../performance-profile/kustomization.yaml | 5 -- .../kustomization.yaml | 1 + .../operator-namespace.yaml | 2 +- .../operator_catalogsource.yaml | 0 .../operator_operatorgroup.yaml | 0 .../operator_subscription.yaml | 0 .../performance_profile.yaml | 0 .../demo/performance-operator/wait_for_it.sh | 8 --- .../performance-profile/kustomization.yaml | 8 --- .../performance}/kustomization.yaml | 4 +- .../operator_catalogsource.patch.yaml | 0 .../performance_profile.patch.yaml | 0 .../performance-operator/wait_for_it.sh | 8 --- .../performance-profile/kustomization.yaml | 8 --- .../performance}/kustomization.yaml | 3 +- .../operator_catalogsource.patch.yaml | 0 .../performance_profile.patch.yaml | 0 hack/feature-deploy.sh | 52 +++++++++++++++---- 19 files changed, 52 insertions(+), 54 deletions(-) delete mode 100644 feature-configs/base/performance-profile/kustomization.yaml rename feature-configs/base/{performance-operator => performance}/kustomization.yaml (88%) rename feature-configs/base/{performance-operator => performance}/operator-namespace.yaml (93%) rename feature-configs/base/{performance-operator => performance}/operator_catalogsource.yaml (100%) rename feature-configs/base/{performance-operator => performance}/operator_operatorgroup.yaml (100%) rename feature-configs/base/{performance-operator => performance}/operator_subscription.yaml (100%) rename feature-configs/base/{performance-profile => performance}/performance_profile.yaml (100%) delete mode 100755 feature-configs/demo/performance-operator/wait_for_it.sh delete mode 100644 feature-configs/demo/performance-profile/kustomization.yaml rename feature-configs/{e2e-gcp/performance-operator => demo/performance}/kustomization.yaml (69%) rename feature-configs/demo/{performance-operator => performance}/operator_catalogsource.patch.yaml (100%) rename feature-configs/demo/{performance-profile => performance}/performance_profile.patch.yaml (100%) delete mode 100755 feature-configs/e2e-gcp/performance-operator/wait_for_it.sh delete mode 100644 feature-configs/e2e-gcp/performance-profile/kustomization.yaml rename feature-configs/{demo/performance-operator => e2e-gcp/performance}/kustomization.yaml (69%) rename feature-configs/e2e-gcp/{performance-operator => performance}/operator_catalogsource.patch.yaml (100%) rename feature-configs/e2e-gcp/{performance-profile => performance}/performance_profile.patch.yaml (100%) diff --git a/Makefile b/Makefile index 2680aea5b8..3c8329d40a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ - #TODO add default features here -export FEATURES?=sctp performance-operator performance-profile +export FEATURES?=sctp performance # The environment represents the kustomize patches to apply when deploying the features export FEATURES_ENVIRONMENT?=e2e-gcp @@ -33,9 +32,9 @@ deps-update: go mod tidy && \ go mod vendor -functests: +functests: feature-deploy @echo "Running Functional Tests" - FEATURES=$(FEATURES) hack/run-functests.sh + FEATURES="$(FEATURES)" hack/run-functests.sh gofmt: @echo "Running gofmt" diff --git a/feature-configs/base/performance-profile/kustomization.yaml b/feature-configs/base/performance-profile/kustomization.yaml deleted file mode 100644 index 0738104a1f..0000000000 --- a/feature-configs/base/performance-profile/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - performance_profile.yaml diff --git a/feature-configs/base/performance-operator/kustomization.yaml b/feature-configs/base/performance/kustomization.yaml similarity index 88% rename from feature-configs/base/performance-operator/kustomization.yaml rename to feature-configs/base/performance/kustomization.yaml index 6a5c4f281b..63ef269558 100644 --- a/feature-configs/base/performance-operator/kustomization.yaml +++ b/feature-configs/base/performance/kustomization.yaml @@ -8,3 +8,4 @@ resources: - operator_operatorgroup.yaml - operator_catalogsource.yaml - operator_subscription.yaml + - performance_profile.yaml diff --git a/feature-configs/base/performance-operator/operator-namespace.yaml b/feature-configs/base/performance/operator-namespace.yaml similarity index 93% rename from feature-configs/base/performance-operator/operator-namespace.yaml rename to feature-configs/base/performance/operator-namespace.yaml index 26cec21269..cbde6e981a 100644 --- a/feature-configs/base/performance-operator/operator-namespace.yaml +++ b/feature-configs/base/performance/operator-namespace.yaml @@ -4,4 +4,4 @@ metadata: labels: openshift.io/cluster-monitoring: "true" name: openshift-performance-addon -spec: {} \ No newline at end of file +spec: {} diff --git a/feature-configs/base/performance-operator/operator_catalogsource.yaml b/feature-configs/base/performance/operator_catalogsource.yaml similarity index 100% rename from feature-configs/base/performance-operator/operator_catalogsource.yaml rename to feature-configs/base/performance/operator_catalogsource.yaml diff --git a/feature-configs/base/performance-operator/operator_operatorgroup.yaml b/feature-configs/base/performance/operator_operatorgroup.yaml similarity index 100% rename from feature-configs/base/performance-operator/operator_operatorgroup.yaml rename to feature-configs/base/performance/operator_operatorgroup.yaml diff --git a/feature-configs/base/performance-operator/operator_subscription.yaml b/feature-configs/base/performance/operator_subscription.yaml similarity index 100% rename from feature-configs/base/performance-operator/operator_subscription.yaml rename to feature-configs/base/performance/operator_subscription.yaml diff --git a/feature-configs/base/performance-profile/performance_profile.yaml b/feature-configs/base/performance/performance_profile.yaml similarity index 100% rename from feature-configs/base/performance-profile/performance_profile.yaml rename to feature-configs/base/performance/performance_profile.yaml diff --git a/feature-configs/demo/performance-operator/wait_for_it.sh b/feature-configs/demo/performance-operator/wait_for_it.sh deleted file mode 100755 index 90408b6d84..0000000000 --- a/feature-configs/demo/performance-operator/wait_for_it.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# Wait for performance-addon-operator deployment to be ready -until ${OC_TOOL} -n openshift-performance-addon get deploy/performance-operator; do - echo "[INFO]: get performance-operator deployment" - sleep 10 -done -${OC_TOOL} -n openshift-performance-addon wait deploy/performance-operator --for condition=Available --timeout 5m diff --git a/feature-configs/demo/performance-profile/kustomization.yaml b/feature-configs/demo/performance-profile/kustomization.yaml deleted file mode 100644 index bd883f3ada..0000000000 --- a/feature-configs/demo/performance-profile/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - ../../base/performance-profile - -patchesStrategicMerge: - - performance_profile.patch.yaml diff --git a/feature-configs/e2e-gcp/performance-operator/kustomization.yaml b/feature-configs/demo/performance/kustomization.yaml similarity index 69% rename from feature-configs/e2e-gcp/performance-operator/kustomization.yaml rename to feature-configs/demo/performance/kustomization.yaml index f628977725..e864238121 100644 --- a/feature-configs/e2e-gcp/performance-operator/kustomization.yaml +++ b/feature-configs/demo/performance/kustomization.yaml @@ -2,8 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../../base/performance-operator + - ../../base/performance patchesStrategicMerge: - operator_catalogsource.patch.yaml - + - performance_profile.patch.yaml diff --git a/feature-configs/demo/performance-operator/operator_catalogsource.patch.yaml b/feature-configs/demo/performance/operator_catalogsource.patch.yaml similarity index 100% rename from feature-configs/demo/performance-operator/operator_catalogsource.patch.yaml rename to feature-configs/demo/performance/operator_catalogsource.patch.yaml diff --git a/feature-configs/demo/performance-profile/performance_profile.patch.yaml b/feature-configs/demo/performance/performance_profile.patch.yaml similarity index 100% rename from feature-configs/demo/performance-profile/performance_profile.patch.yaml rename to feature-configs/demo/performance/performance_profile.patch.yaml diff --git a/feature-configs/e2e-gcp/performance-operator/wait_for_it.sh b/feature-configs/e2e-gcp/performance-operator/wait_for_it.sh deleted file mode 100755 index 90408b6d84..0000000000 --- a/feature-configs/e2e-gcp/performance-operator/wait_for_it.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# Wait for performance-addon-operator deployment to be ready -until ${OC_TOOL} -n openshift-performance-addon get deploy/performance-operator; do - echo "[INFO]: get performance-operator deployment" - sleep 10 -done -${OC_TOOL} -n openshift-performance-addon wait deploy/performance-operator --for condition=Available --timeout 5m diff --git a/feature-configs/e2e-gcp/performance-profile/kustomization.yaml b/feature-configs/e2e-gcp/performance-profile/kustomization.yaml deleted file mode 100644 index bd883f3ada..0000000000 --- a/feature-configs/e2e-gcp/performance-profile/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - ../../base/performance-profile - -patchesStrategicMerge: - - performance_profile.patch.yaml diff --git a/feature-configs/demo/performance-operator/kustomization.yaml b/feature-configs/e2e-gcp/performance/kustomization.yaml similarity index 69% rename from feature-configs/demo/performance-operator/kustomization.yaml rename to feature-configs/e2e-gcp/performance/kustomization.yaml index e754a456f9..e864238121 100644 --- a/feature-configs/demo/performance-operator/kustomization.yaml +++ b/feature-configs/e2e-gcp/performance/kustomization.yaml @@ -2,7 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../../base/performance-operator + - ../../base/performance patchesStrategicMerge: - operator_catalogsource.patch.yaml + - performance_profile.patch.yaml diff --git a/feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml b/feature-configs/e2e-gcp/performance/operator_catalogsource.patch.yaml similarity index 100% rename from feature-configs/e2e-gcp/performance-operator/operator_catalogsource.patch.yaml rename to feature-configs/e2e-gcp/performance/operator_catalogsource.patch.yaml diff --git a/feature-configs/e2e-gcp/performance-profile/performance_profile.patch.yaml b/feature-configs/e2e-gcp/performance/performance_profile.patch.yaml similarity index 100% rename from feature-configs/e2e-gcp/performance-profile/performance_profile.patch.yaml rename to feature-configs/e2e-gcp/performance/performance_profile.patch.yaml diff --git a/hack/feature-deploy.sh b/hack/feature-deploy.sh index a0f894dce6..98e8070107 100755 --- a/hack/feature-deploy.sh +++ b/hack/feature-deploy.sh @@ -24,16 +24,50 @@ node=$(${OC_TOOL} get nodes --selector='node-role.kubernetes.io/worker' -o name ${OC_TOOL} label --overwrite=true $node node-role.kubernetes.io/worker-rt="" # Deploy features -for feature in $FEATURES; do +success=0 +iterations=0 +sleep_time=10 +max_iterations=30 # results in 5 minute timeout +until [[ $success -eq 1 ]] || [[ $iterations -eq $max_iterations ]] +do - echo "[INFO]: Deploying feature '$feature' for environment '$FEATURES_ENVIRONMENT'" - ${KUSTOMIZE} build feature-configs/${FEATURES_ENVIRONMENT}/${feature}/ | ${OC_TOOL} apply -f - + feature_failed=0 - # Wait for feature - if [ -f "feature-configs/${FEATURES_ENVIRONMENT}/${feature}/wait_for_it.sh" ]; then - echo "[INFO]: waiting for $feature to be deployed" - feature-configs/${FEATURES_ENVIRONMENT}/${feature}/wait_for_it.sh - echo "[INFO]: $feature was deployed" - fi; + for feature in $FEATURES; do + + feature_dir=feature-configs/${FEATURES_ENVIRONMENT}/${feature}/ + if [[ ! -d $feature_dir ]]; then + echo "[WARN] Feature '$feature' is not configured for environment '$FEATURES_ENVIRONMENT', skipping it" + continue + fi + + echo "[INFO] Deploying feature '$feature' for environment '$FEATURES_ENVIRONMENT'" + set +e + if ! ${KUSTOMIZE} build $feature_dir | ${OC_TOOL} apply -f - + then + echo "[WARN] Deployment of feature '$feature' failed." + feature_failed=1 + fi + set -e + + done + + if [[ $feature_failed -eq 1 ]]; then + iterations=$((iterations + 1)) + iterations_left=$((max_iterations - iterations)) + echo "[WARN] At least one deployment failed, retrying in $sleep_time sec, $iterations_left retries left" + sleep $sleep_time + continue + + fi + + # All features deployed successfully + success=1 done + +if [[ $success -eq 1 ]]; then + echo "[INFO] Deployment successful" +else + echo "[ERROR] Deployment failed" +fi From e2b912d9e321c012be5be07ca1f0c193b5edcc6a Mon Sep 17 00:00:00 2001 From: Marc Sluiter Date: Tue, 14 Jan 2020 10:34:00 +0100 Subject: [PATCH 4/4] Separated worker-rt labeling and deployment, improved README Signed-off-by: Marc Sluiter --- Makefile | 6 +++++- README.md | 15 ++++++++++++++- hack/feature-deploy.sh | 5 ----- hack/label-worker-rt.sh | 11 +++++++++++ 4 files changed, 30 insertions(+), 7 deletions(-) create mode 100755 hack/label-worker-rt.sh diff --git a/Makefile b/Makefile index 3c8329d40a..f41bfee950 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ deps-update: go mod tidy && \ go mod vendor -functests: feature-deploy +functests: label-worker-rt feature-deploy @echo "Running Functional Tests" FEATURES="$(FEATURES)" hack/run-functests.sh @@ -64,3 +64,7 @@ kustomize: feature-deploy: kustomize KUSTOMIZE=$(KUSTOMIZE) FEATURES_ENVIRONMENT=$(FEATURES_ENVIRONMENT) FEATURES="$(FEATURES)" hack/feature-deploy.sh + +label-worker-rt: + @echo "Adding worker-rt label to one worker node" + hack/label-worker-rt.sh diff --git a/README.md b/README.md index d7db0cbd10..be72a0a08f 100644 --- a/README.md +++ b/README.md @@ -4,4 +4,17 @@ This repo contains example kustomize configs used to installed openshift feature # Contributing kustomize configs -All kustomize configs should be entirely declarative in nature. This means no bash plugin modules performing imparative tasks. Features should be installed simply by posting manifests to the cluster. After posting manifests, determining when the cluster has converged on those manifests successully should be observable. +All kustomize configs should be entirely declarative in nature. This means no bash plugin modules performing imparative tasks. Features should be installed simply by posting manifests to the cluster. After posting manifests, determining when the cluster has converged on those manifests successully should be observable. + +# Usage + +## Prerequisites + +- You need a running OCP 4.4 cluster and a valid KUBECONFIG. +- You need at least one node with the `node-role.kubernetes.io/worker-rt=""` label. + Run `make label-worker-rt` for adding it on the first `worker` node of the cluster. + +## Deployment + +- run `FEATURES_ENVIRONMENT=demo make feature-deploy`. + This will try to apply all manifests in a loop until all deployments succeeded, or until it runs into a timeout. diff --git a/hack/feature-deploy.sh b/hack/feature-deploy.sh index 98e8070107..cd73d6e12b 100755 --- a/hack/feature-deploy.sh +++ b/hack/feature-deploy.sh @@ -18,11 +18,6 @@ export OC_TOOL="${OC_TOOL:-oc}" # expect kustomize to be in PATH by default KUSTOMIZE="${KUSTOMIZE:-kustomize}" -# Label 1 worker node -echo "[INFO]:labeling 1 worker node with worker-rt" -node=$(${OC_TOOL} get nodes --selector='node-role.kubernetes.io/worker' -o name | head -1) -${OC_TOOL} label --overwrite=true $node node-role.kubernetes.io/worker-rt="" - # Deploy features success=0 iterations=0 diff --git a/hack/label-worker-rt.sh b/hack/label-worker-rt.sh new file mode 100755 index 0000000000..e192eb8be0 --- /dev/null +++ b/hack/label-worker-rt.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +# expect oc to be in PATH by default +export OC_TOOL="${OC_TOOL:-oc}" + +# Label 1 worker node +echo "[INFO]: Labeling 1 worker node with worker-rt" +node=$(${OC_TOOL} get nodes --selector='node-role.kubernetes.io/worker' -o name | head -1) +${OC_TOOL} label $node node-role.kubernetes.io/worker-rt=""