Add monitoring operator in the CVO overrides before the cluster starts

createdisk.sh

@@ -19,10 +19,9 @@ if [[ ${OKD_VERSION} != "none" ]]
 then
     OPENSHIFT_VERSION=${OKD_VERSION}
     BASE_OS=fedora-coreos
-    USE_LUKS=false
 fi
 BASE_OS=${BASE_OS:-rhcos}
-USE_LUKS=${USE_LUKS:-true}
+USE_LUKS=${USE_LUKS:-false}
 
 # CRC_VM_NAME: short VM name to use in crc_libvirt.sh
 # BASE_DOMAIN: domain used for the cluster

cvo-overrides-after-first-run.yaml

@@ -0,0 +1,110 @@
+- op: add
+  path: /spec/overrides
+  value:
+  - kind: Deployment
+    group: apps/v1
+    name: cluster-monitoring-operator
+    namespace: openshift-monitoring
+    unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: monitoring
+    namespace: ""
+    unmanaged: true
+  - kind: Deployment
+    group: apps/v1
+    name: kube-storage-version-migrator-operator
+    namespace: openshift-kube-storage-version-migrator-operator
+    unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: kube-storage-version-migrator
+    namespace: ""
+    unmanaged: true
+  - kind: Deployment
+    group: apps/v1
+    name: insights-operator
+    namespace: openshift-insights
+    unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: insights
+    namespace: ""
+    unmanaged: true
+  - kind: Deployment
+    group: apps/v1
+    name: cloud-credential-operator
+    namespace: openshift-cloud-credential-operator
+    unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: cloud-credential
+    namespace: ""
+    unmanaged: true
+  - kind: Deployment
+    group: apps/v1
+    name: cluster-storage-operator
+    namespace: openshift-cluster-storage-operator
+    unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: storage
+    namespace: ""
+    unmanaged: true
+  - kind: Deployment
+    group: apps/v1
+    name: cluster-baremetal-operator
+    namespace: openshift-machine-api
+    unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: baremetal
+    namespace: ""
+    unmanaged: true
+  - kind: Deployment
+    group: apps/v1
+    name: cluster-autoscaler-operator
+    namespace: openshift-machine-api
+    unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: cluster-autoscaler
+    namespace: ""
+    unmanaged: true
+  - kind: Deployment
+    group: apps/v1
+    name: csi-snapshot-controller-operator
+    namespace: openshift-cluster-storage-operator
+    unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: csi-snapshot-controller
+    namespace: ""
+    unmanaged: true
+  # only used in bootstrap phase
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: machine-api
+    namespace: ""
+    unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: machine-config
+    namespace: ""
+    unmanaged: true
+  - kind: Deployment
+    group: apps/v1
+    name: machine-config-operator
+    namespace: openshift-machine-config-operator
+    unmanaged: true
+  - kind: Deployment
+    group: apps/v1
+    name: machine-api-operator
+    namespace: openshift-machine-api
+    unmanaged: true
+  # required to scale down etcd-quorum-guard
+  - kind: Deployment
+    group: apps/v1
+    name: etcd-quorum-guard
+    namespace: openshift-etcd
+    unmanaged: true

cvo_override.yaml → cvo-overrides.yaml

@@ -1,63 +1,82 @@
-- op: add
-  path: /spec/overrides
-  value:
+spec:
+  overrides:
   - kind: Deployment
     group: apps/v1
     name: cluster-monitoring-operator
     namespace: openshift-monitoring
     unmanaged: true
-  - kind: Deployment
-    group: apps/v1
-    name: machine-config-operator
-    namespace: openshift-machine-config-operator
-    unmanaged: true
-  - kind: Deployment
-    group: apps/v1
-    name: etcd-quorum-guard
-    namespace: openshift-machine-config-operator
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: monitoring
+    namespace: ""
     unmanaged: true
   - kind: Deployment
     group: apps/v1
-    name: machine-api-operator
-    namespace: openshift-machine-api
+    name: kube-storage-version-migrator-operator
+    namespace: openshift-kube-storage-version-migrator-operator
     unmanaged: true
-  - kind: Deployment
-    group: apps/v1
-    name: cluster-autoscaler-operator
-    namespace: openshift-machine-api
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: kube-storage-version-migrator
+    namespace: ""
     unmanaged: true
   - kind: Deployment
     group: apps/v1
     name: insights-operator
     namespace: openshift-insights
     unmanaged: true
-  - kind: Deployment
-    group: apps/v1
-    name: prometheus-k8s
-    namespace: openshift-cloud-credential-operator
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: insights
+    namespace: ""
     unmanaged: true
   - kind: Deployment
     group: apps/v1
     name: cloud-credential-operator
     namespace: openshift-cloud-credential-operator
     unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: cloud-credential
+    namespace: ""
+    unmanaged: true
   - kind: Deployment
     group: apps/v1
-    name: csi-snapshot-controller-operator
+    name: cluster-storage-operator
     namespace: openshift-cluster-storage-operator
     unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: storage
+    namespace: ""
+    unmanaged: true
   - kind: Deployment
     group: apps/v1
-    name: cluster-storage-operator
-    namespace: openshift-cluster-storage-operator
+    name: cluster-baremetal-operator
+    namespace: openshift-machine-api
+    unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: baremetal
+    namespace: ""
     unmanaged: true
   - kind: Deployment
     group: apps/v1
-    name: kube-storage-version-migrator-operator
-    namespace: openshift-kube-storage-version-migrator-operator
+    name: cluster-autoscaler-operator
+    namespace: openshift-machine-api
+    unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: cluster-autoscaler
+    namespace: ""
     unmanaged: true
   - kind: Deployment
     group: apps/v1
-    name: etcd-quorum-guard
-    namespace: openshift-etcd
+    name: csi-snapshot-controller-operator
+    namespace: openshift-cluster-storage-operator
+    unmanaged: true
+  - kind: ClusterOperator
+    group: config.openshift.io/v1
+    name: csi-snapshot-controller
+    namespace: ""
     unmanaged: true

snc.sh

@@ -25,7 +25,7 @@ CRC_VM_NAME=${CRC_VM_NAME:-crc}
 BASE_DOMAIN=${CRC_BASE_DOMAIN:-testing}
 CRC_PV_DIR="/mnt/pv-data"
 SSH="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i id_ecdsa_crc"
-MIRROR=${MIRROR:-https://mirror.openshift.com/pub/openshift-v4/$ARCH/clients/ocp}
+MIRROR=${MIRROR:-https://mirror.openshift.com/pub/openshift-v4/clients/ocp-dev-preview/pre-release}
 CERT_ROTATION=${SNC_DISABLE_CERT_ROTATION:-enabled}
 
 # If user defined the OPENSHIFT_VERSION environment variable then use it.
@@ -34,7 +34,7 @@ if test -n "${OPENSHIFT_VERSION-}"; then
     OPENSHIFT_RELEASE_VERSION=${OPENSHIFT_VERSION}
     echo "Using release ${OPENSHIFT_RELEASE_VERSION} from OPENSHIFT_VERSION"
 else
-    OPENSHIFT_RELEASE_VERSION="$(curl -L "${MIRROR}"/candidate-4.6/release.txt | sed -n 's/^ *Version: *//p')"
+    OPENSHIFT_RELEASE_VERSION="$(curl -L "${MIRROR}"/release.txt | sed -n 's/^ *Version: *//p')"
     if test -n "${OPENSHIFT_RELEASE_VERSION}"; then
         echo "Using release ${OPENSHIFT_RELEASE_VERSION} from the latest mirror"
     else
@@ -47,13 +47,13 @@ fi
 mkdir -p openshift-clients/linux openshift-clients/mac openshift-clients/windows
 if [[ ${OKD_VERSION} != "none" ]]
 then
-    curl -L "${MIRROR}/${OPENSHIFT_RELEASE_VERSION}/openshift-client-linux-${OPENSHIFT_RELEASE_VERSION}.tar.gz" | tar -zx -C openshift-clients/linux oc
-    curl -L "${MIRROR}/${OPENSHIFT_RELEASE_VERSION}/openshift-client-mac-${OPENSHIFT_RELEASE_VERSION}.tar.gz" | tar -zx -C openshift-clients/mac oc
-    curl -L "${MIRROR}/${OPENSHIFT_RELEASE_VERSION}/openshift-client-windows-${OPENSHIFT_RELEASE_VERSION}.zip" > openshift-clients/windows/oc.zip
+    curl -L "${MIRROR}/openshift-client-linux-${OPENSHIFT_RELEASE_VERSION}.tar.gz" | tar -zx -C openshift-clients/linux oc
+    curl -L "${MIRROR}/openshift-client-mac-${OPENSHIFT_RELEASE_VERSION}.tar.gz" | tar -zx -C openshift-clients/mac oc
+    curl -L "${MIRROR}/openshift-client-windows-${OPENSHIFT_RELEASE_VERSION}.zip" > openshift-clients/windows/oc.zip
 else
-    curl -L "${MIRROR}/${OPENSHIFT_RELEASE_VERSION}/openshift-client-linux.tar.gz" | tar -zx -C openshift-clients/linux oc
-    curl -L "${MIRROR}/${OPENSHIFT_RELEASE_VERSION}/openshift-client-mac.tar.gz" | tar -zx -C openshift-clients/mac oc
-    curl -L "${MIRROR}/${OPENSHIFT_RELEASE_VERSION}/openshift-client-windows.zip" > openshift-clients/windows/oc.zip
+    curl -L "${MIRROR}/openshift-client-linux.tar.gz" | tar -zx -C openshift-clients/linux oc
+    curl -L "${MIRROR}/openshift-client-mac.tar.gz" | tar -zx -C openshift-clients/mac oc
+    curl -L "${MIRROR}/openshift-client-windows.zip" > openshift-clients/windows/oc.zip
 fi
 ${UNZIP} -o -d openshift-clients/windows/ openshift-clients/windows/oc.zip
 OC=./openshift-clients/linux/oc
@@ -69,7 +69,7 @@ elif [ ! -f ${OPENSHIFT_PULL_SECRET_PATH} ]; then
 fi
 
 if test -z "${OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE-}"; then
-    OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="$(curl -L "${MIRROR}/${OPENSHIFT_RELEASE_VERSION}/release.txt" | sed -n 's/^Pull From: //p')"
+    OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="$(curl -L "${MIRROR}/release.txt" | sed -n 's/^Pull From: //p')"
 elif test -n "${OPENSHIFT_VERSION-}"; then
     echo "Both OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE and OPENSHIFT_VERSION are set, OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE will take precedence"
     echo "OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE"
@@ -145,6 +145,9 @@ ${YQ} write --inplace ${INSTALL_DIR}/install-config.yaml sshKey "$(cat id_ecdsa_
 # Create the manifests using the INSTALL_DIR
 OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE=$OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE ${OPENSHIFT_INSTALL} --dir ${INSTALL_DIR} create manifests || exit 1
 
+# Add CVO overrides before first start of the cluster. Objects declared in this file won't be created.
+${YQ} merge -i ${INSTALL_DIR}/manifests/cvo-overrides.yaml cvo-overrides.yaml
+
 # Add custom domain to cluster-ingress
 ${YQ} write --inplace ${INSTALL_DIR}/manifests/cluster-ingress-02-config.yml spec[domain] apps-${CRC_VM_NAME}.${BASE_DOMAIN}
 # Add master memory to 12 GB and 6 cpus 
@@ -179,59 +182,30 @@ fi
 # Wait for install to complete, this provide another 30 mins to make resources (apis) stable
 OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE=$OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE ${OPENSHIFT_INSTALL} --dir ${INSTALL_DIR} wait-for install-complete ${OPENSHIFT_INSTALL_EXTRA_ARGS}
 
-
 # Set the VM static hostname to crc-xxxxx-master-0 instead of localhost.localdomain
 HOSTNAME=$(${SSH} core@api.${CRC_VM_NAME}.${BASE_DOMAIN} hostnamectl status --transient)
 ${SSH} core@api.${CRC_VM_NAME}.${BASE_DOMAIN} sudo hostnamectl set-hostname ${HOSTNAME}
 
 create_json_description
 
-
 # Create persistent volumes
 create_pvs "${CRC_PV_DIR}" 30
 
 # Mark some of the deployments unmanaged by the cluster-version-operator (CVO)
 # https://github.com/openshift/cluster-version-operator/blob/master/docs/dev/clusterversion.md#setting-objects-unmanaged
-retry ${OC} patch clusterversion version --type json -p "$(cat cvo_override.yaml)"
-
-# Clean-up 'openshift-monitoring' namespace
-delete_operator "deployment/cluster-monitoring-operator" "openshift-monitoring" "app=cluster-monitoring-operator"
-delete_operator "deployment/prometheus-operator" "openshift-monitoring" "app.kubernetes.io/name=prometheus-operator"
-delete_operator "deployment/prometheus-adapter" "openshift-monitoring" "name=prometheus-adapter"
-delete_operator "statefulset/alertmanager-main" "openshift-monitoring" "app=alertmanager"
-retry ${OC} delete statefulset,deployment,daemonset --all -n openshift-monitoring
-# Delete prometheus rule application webhook
-retry ${OC} delete validatingwebhookconfigurations prometheusrules.openshift.io
-
-# Delete the pods which are there in Complete state
-retry ${OC} delete pods -l 'app in (installer, pruner)' -n openshift-kube-apiserver
-retry ${OC} delete pods -l 'app in (installer, pruner)' -n openshift-kube-scheduler
-retry ${OC} delete pods -l 'app in (installer, pruner)' -n openshift-kube-controller-manager
+# Objects declared in this file are still created by the CVO at startup.
+# The CVO won't modify these objects anymore with the following command. Hence, we can remove them afterwards.
+retry ${OC} patch clusterversion version --type json -p "$(cat cvo-overrides-after-first-run.yaml)"
 
 # Clean-up 'openshift-machine-api' namespace
 delete_operator "deployment/machine-api-operator" "openshift-machine-api" "k8s-app=machine-api-operator"
 retry ${OC} delete statefulset,deployment,daemonset --all -n openshift-machine-api
+retry ${OC} delete clusteroperator machine-api
 
 # Clean-up 'openshift-machine-config-operator' namespace
 delete_operator "deployment/machine-config-operator" "openshift-machine-config-operator" "k8s-app=machine-config-operator"
 retry ${OC} delete statefulset,deployment,daemonset --all -n openshift-machine-config-operator
-
-# Clean-up 'openshift-insights' namespace
-retry ${OC} delete statefulset,deployment,daemonset --all -n openshift-insights
-
-# Clean-up 'openshift-cloud-credential-operator' namespace
-retry ${OC} delete statefulset,deployment,daemonset --all -n openshift-cloud-credential-operator
-
-# Clean-up 'openshift-cluster-storage-operator' namespace
-delete_operator "deployment.apps/csi-snapshot-controller-operator" "openshift-cluster-storage-operator" "app=csi-snapshot-controller-operator"
-retry ${OC} delete statefulset,deployment,daemonset --all -n openshift-cluster-storage-operator
-
-# Clean-up 'openshift-kube-storage-version-migrator-operator' namespace
-retry ${OC} delete statefulset,deployment,daemonset --all -n openshift-kube-storage-version-migrator-operator
-
-# Delete the v1beta1.metrics.k8s.io apiservice since we are already scale down cluster wide monitioring.
-# Since this CRD block namespace deletion forever.
-retry ${OC} delete apiservice v1beta1.metrics.k8s.io
+retry ${OC} delete clusteroperator machine-config
 
 # Scale route deployment from 2 to 1
 retry ${OC} scale --replicas=1 ingresscontroller/default -n openshift-ingress-operator
@@ -241,3 +215,6 @@ retry ${OC} scale --replicas=1 deployment etcd-quorum-guard -n openshift-etcd
 
 # Set default route for registry CRD from false to true.
 retry ${OC} patch config.imageregistry.operator.openshift.io/cluster --patch '{"spec":{"defaultRoute":true}}' --type=merge
+
+# Delete the pods which are there in Complete state
+retry ${OC} delete pod --field-selector=status.phase==Succeeded --all-namespaces