OCPQE-19978: Add CloudControllerManager and Ingress Cap for v416

ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16__multi-nightly.yaml

@@ -313,7 +313,8 @@ tests:
     dependencies:
       OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest
     env:
-      ADDITIONAL_ENABLED_CAPABILITIES: MachineAPI CloudCredential
+      ADDITIONAL_ENABLED_CAPABILITIES: MachineAPI CloudCredential CloudControllerManager
+        Ingress
       BASE_DOMAIN: qe.devcluster.openshift.com
       BASELINE_CAPABILITY_SET: None
       COMPUTE_NODE_TYPE: m6g.xlarge
@@ -335,7 +336,8 @@ tests:
     dependencies:
       OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest
     env:
-      ADDITIONAL_ENABLED_CAPABILITIES: MachineAPI CloudCredential
+      ADDITIONAL_ENABLED_CAPABILITIES: MachineAPI CloudCredential CloudControllerManager
+        Ingress
       BASE_DOMAIN: qe.devcluster.openshift.com
       BASELINE_CAPABILITY_SET: None
       E2E_RUN_TAGS: '@disconnected and not @workloads'
@@ -699,7 +701,7 @@ tests:
     dependencies:
       OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest
     env:
-      ADDITIONAL_ENABLED_CAPABILITIES: CloudCredential
+      ADDITIONAL_ENABLED_CAPABILITIES: CloudCredential CloudControllerManager Ingress
       BASE_DOMAIN: qe.devcluster.openshift.com
       BASELINE_CAPABILITY_SET: None
       E2E_RUN_TAGS: not @workloads
@@ -714,7 +716,7 @@ tests:
     dependencies:
       OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest
     env:
-      ADDITIONAL_ENABLED_CAPABILITIES: CloudCredential
+      ADDITIONAL_ENABLED_CAPABILITIES: CloudCredential CloudControllerManager Ingress
       BASE_DOMAIN: qe.devcluster.openshift.com
       BASELINE_CAPABILITY_SET: None
       BOOTSTRAP_INSTANCE_TYPE: m6g.large
@@ -792,7 +794,7 @@ tests:
     dependencies:
       OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest
     env:
-      ADDITIONAL_ENABLED_CAPABILITIES: CloudCredential
+      ADDITIONAL_ENABLED_CAPABILITIES: CloudCredential CloudControllerManager Ingress
       BASE_DOMAIN: qe.azure.devcluster.openshift.com
       E2E_RUN_TAGS: not @workloads
       PERSISTENT_MONITORING: "false"
@@ -806,7 +808,7 @@ tests:
     dependencies:
       OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest
     env:
-      ADDITIONAL_ENABLED_CAPABILITIES: CloudCredential
+      ADDITIONAL_ENABLED_CAPABILITIES: CloudCredential CloudControllerManager Ingress
       BASE_DOMAIN: qe.azure.devcluster.openshift.com
       COMPUTE_NODE_TYPE: Standard_D4ps_v5
       E2E_RUN_TAGS: not @workloads
@@ -1404,6 +1406,28 @@ tests:
     test:
     - chain: openshift-e2e-test-qe
     workflow: baremetal-lab-agent-install-network-dynamic-disconnected-day2
+- as: baremetal-ipi-ovn-ipv4-basecap-none-f28
+  cluster: build05
+  cron: 12 15 29 * *
+  steps:
+    cluster_profile: equinix-ocp-metal-qe
+    dependencies:
+      MULTI_RELEASE_IMAGE: release:multi-latest
+      OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest
+    env:
+      ADDITIONAL_ENABLED_CAPABILITIES: baremetal MachineAPI Ingress
+      AUX_HOST: openshift-qe-metal-ci.arm.eng.rdu2.redhat.com
+      BASELINE_CAPABILITY_SET: None
+      DISCONNECTED: "false"
+      PROVISIONING_HOST: openshift-qe-metal-ci.arm.eng.rdu2.redhat.com
+      PROVISIONING_NET_DEV: enP2p1s0f0np0
+      RESERVE_BOOTSTRAP: "false"
+      architecture: arm64
+      masters: "3"
+      workers: "2"
+    test:
+    - chain: openshift-e2e-test-qe
+    workflow: baremetal-lab-ipi
 - as: baremetal-ipi-ovn-ipv4-external-lb-f7
   cluster: build05
   cron: 41 18 7,14,21,28 * *

ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16-periodics.yaml

@@ -48741,6 +48741,95 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build05
+  cron: 12 15 29 * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-4.16
+    org: openshift
+    repo: openshift-tests-private
+  labels:
+    ci-operator.openshift.io/cloud: equinix-ocp-metal
+    ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-metal-qe
+    ci-operator.openshift.io/cluster: build05
+    ci-operator.openshift.io/variant: multi-nightly
+    ci.openshift.io/generator: prowgen
+    job-release: "4.16"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-openshift-tests-private-release-4.16-multi-nightly-baremetal-ipi-ovn-ipv4-basecap-none-f28
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --oauth-token-path=/usr/local/github-credentials/oauth
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --secret-dir=/usr/local/baremetal-ipi-ovn-ipv4-basecap-none-f28-cluster-profile
+      - --target=baremetal-ipi-ovn-ipv4-basecap-none-f28
+      - --variant=multi-nightly
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /usr/local/baremetal-ipi-ovn-ipv4-basecap-none-f28-cluster-profile
+        name: cluster-profile
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /usr/local/github-credentials
+        name: github-credentials-openshift-ci-robot-private-git-cloner
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: cluster-profile
+      secret:
+        secretName: cluster-secrets-equinix-ocp-metal-qe
+    - name: github-credentials-openshift-ci-robot-private-git-cloner
+      secret:
+        secretName: github-credentials-openshift-ci-robot-private-git-cloner
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build05
   cron: 41 18 7,14,21,28 * *

ci-operator/step-registry/cucushift/installer/check/capability/cucushift-installer-check-capability-commands.sh

@@ -88,6 +88,8 @@ caps_operator[MachineAPI]="machine-api control-plane-machine-set cluster-autosca
 caps_operator[ImageRegistry]="image-registry"
 caps_operator[OperatorLifecycleManager]="operator-lifecycle-manager operator-lifecycle-manager-catalog operator-lifecycle-manager-packageserver"
 caps_operator[CloudCredential]="cloud-credential"
+caps_operator[CloudControllerManager]="cloud-controller-manager"
+caps_operator[Ingress]="ingress"
 
 # Mapping between optional capability and resources
 # Need to be updated when new resource marks as optional
@@ -102,8 +104,12 @@ v412=" ${v411} Console Insights Storage CSISnapshot"
 v413=" ${v412} NodeTuning"
 v414=" ${v413} MachineAPI Build DeploymentConfig ImageRegistry"
 v415=" ${v414} OperatorLifecycleManager CloudCredential"
-latest_defined="v415"
+v416=" ${v415} CloudControllerManager Ingress"
+latest_defined="v416"
 always_default="${!latest_defined}"
+# always enabled capabilities
+#declare -A always_enabled_caps
+#always_enabled_caps[416]="Ingress"
 
 # Determine vCurrent
 declare "v${ocp_major_version}${ocp_minor_version}"
@@ -140,6 +146,9 @@ case ${baselinecaps_from_config} in
 "v4.15")
   enabled_capability_set="${v415}"
   ;;
+"v4.16")
+  enabled_capability_set="${v416}"
+  ;;
 "vCurrent")
   enabled_capability_set="${vCurrent}"
   ;;
@@ -152,6 +161,12 @@ additional_caps_from_config=$(yq-go r "${SHARED_DIR}/install-config.yaml" "capab
 if [[ "${additional_caps_from_config}" != "" ]]; then
     enabled_capability_set="${enabled_capability_set} ${additional_caps_from_config}"
 fi
+# Once pr https://github.com/openshift/cluster-version-operator/pull/946 merged, the code can be opened
+#for version in "${!always_enabled_caps[@]}"; do
+#    if [[ ${ocp_version/.} -ge ${version} ]]; then
+#        enabled_capability_set="${enabled_capability_set} ${always_enabled_caps[$version]}"
+#    fi
+#done
 enabled_capability_set=$(echo ${enabled_capability_set} | xargs -n1 | sort -u | xargs)
 disabled_capability_set="${vCurrent}"
 for cap in $enabled_capability_set; do

ci-operator/step-registry/cucushift/upgrade/check/capability/cucushift-upgrade-check-capability-commands.sh

@@ -71,8 +71,11 @@ function get_caps_for_version_capset() {
     "v4.15")
     caps_string="baremetal marketplace openshift-samples Console Insights Storage CSISnapshot NodeTuning MachineAPI Build DeploymentConfig ImageRegistry OperatorLifecycleManager CloudCredential"
     ;;
+    "v4.16")
+    caps_string="baremetal marketplace openshift-samples Console Insights Storage CSISnapshot NodeTuning MachineAPI Build DeploymentConfig ImageRegistry OperatorLifecycleManager CloudCredential CloudControllerManager Ingress"
+    ;;
     *)
-    caps_string="baremetal marketplace openshift-samples Console Insights Storage CSISnapshot NodeTuning MachineAPI Build DeploymentConfig ImageRegistry OperatorLifecycleManager CloudCredential"
+    caps_string="baremetal marketplace openshift-samples Console Insights Storage CSISnapshot NodeTuning MachineAPI Build DeploymentConfig ImageRegistry OperatorLifecycleManager CloudCredential CloudControllerManager Ingress"
     ;;
     esac
 
@@ -122,6 +125,12 @@ function get_actual_implicit_caps {
     # The condition message could be "The following capabilities could not be disabled: Console, Insights, Storage"
     tmp_capabilities=$(echo "$implicit_message" | grep -oE 'could not be disabled: (.*)' | cut -d':' -f2)
     actual_implicit_caps=$(echo "$tmp_capabilities" | awk -F', ' '{ for (i=1; i<=NF; i++) print $i }')
+    # need to remove once pr https://github.com/openshift/cluster-version-operator/pull/946 is merged
+    for version in "${!always_enabled_caps[@]}"; do
+        if [[ ${ocp_version/.} -ge ${version} ]]; then
+            actual_implicit_caps="${actual_implicit_caps} ${always_enabled_caps[$version]}"
+        fi
+    done
     # echo "Actual implicitly enabled capabilities list is ${actual_implicit_caps}"
 }
 
@@ -215,6 +224,14 @@ function check_cvo_cap() {
             result=1
         else
             cvo_caps_str=$(echo $cvo_caps | tr -d '["]' | tr "," " " | xargs -n1 | sort -u | xargs)
+            # need to remove once pr https://github.com/openshift/cluster-version-operator/pull/946 is merged
+            for version in "${!always_enabled_caps[@]}"; do
+                if [[ ${ocp_version/.} -ge ${version} ]]; then
+                    cvo_caps_str="${cvo_caps_str} ${always_enabled_caps[$version]}"
+                fi
+            done
+            cvo_caps_str=$(echo ${cvo_caps_str} | xargs -n1 | sort -u | xargs)
+
             if [[ "${cvo_caps_str}" == "${capability_set}" ]]; then
                 echo "INFO: ${expected_status} capabilities matches with cvo ${cvo_field}!"
                 echo -e "cvo_caps: ${cvo_caps_str}\n${expected_status} capability set: ${capability_set}"
@@ -251,6 +268,11 @@ if [[ -z "${baselinecaps_from_cluster}" || "${baselinecaps_from_cluster}" == "nu
 fi
 echo "baselinecaps_from_cluster: ${baselinecaps_from_cluster}"
 
+ocp_version=$(oc version -o json | jq -r '.openshiftVersion' | cut -d '.' -f1,2 | tr -d '.')
+# always enabled capabilities
+declare -A always_enabled_caps
+always_enabled_caps[416]="Ingress"
+
 # shellcheck disable=SC2207
 version_set=($(oc get clusterversion version -ojson | jq -r .status.history[].version))
 
@@ -269,6 +291,8 @@ caps_operator[MachineAPI]="machine-api control-plane-machine-set cluster-autosca
 caps_operator[ImageRegistry]="image-registry"
 caps_operator[OperatorLifecycleManager]="operator-lifecycle-manager operator-lifecycle-manager-catalog operator-lifecycle-manager-packageserver"
 caps_operator[CloudCredential]="cloud-credential"
+caps_operator[CloudControllerManager]="cloud-controller-manager"
+caps_operator[Ingress]="ingress"
 
 # Mapping between optional capability and resources
 # Need update when new resource marks as optional

ci-operator/step-registry/ipi/conf/capability/additionalcaps/ipi-conf-capability-additionalcaps-commands.sh

@@ -60,11 +60,11 @@ ocp_minor_version=$( echo "${ocp_version}" | awk --field-separator=. '{print $2}
 v411="baremetal marketplace openshift-samples"
 v412=" ${v411} Console Insights Storage CSISnapshot"
 v413=" ${v412} NodeTuning"
-# shellcheck disable=SC2034
 v414=" ${v413} MachineAPI Build DeploymentConfig ImageRegistry"
-# shellcheck disable=SC2034
 v415=" ${v414} OperatorLifecycleManager CloudCredential"
-latest_version="v415"
+# shellcheck disable=SC2034
+v416=" ${v415} CloudControllerManager Ingress"
+latest_version="v416"
 
 # define capability dependency
 declare -A dependency_caps
@@ -104,6 +104,16 @@ while [[ -z "${selected_capability}" ]]; do
         echo "WARNING: MachineAPI is selected, but it requires on IPI, could not be disabled!"
         selected_capability=""
         ;;
+    "CloudControllerManager")
+        platform_type=$(yq-go r -j "${SHARED_DIR}/install-config.yaml" "platform" | jq -r 'keys[]')
+        ccm_type=$(yq-go r "${SHARED_DIR}/install-config.yaml" "platform.external.cloudControllerManager")
+        if [[ "${platform_type}" == "none" || "${platform_type}" == "baremetal" || "${ccm_type}" == "None" ]]; then
+            enabled_capabilities=${enabled_capabilities/${selected_capability}}
+        else
+            echo "WARNING: CloudControllerManager is selected, but it is required on Cloud platform and External platform with cloudControllerManager state External, could not be disabled!"
+            selected_capability=""
+        fi
+        ;;
     # To be updated once OCP 4.16 is released
     "CloudCredential")
         if [[ "${CLUSTER_TYPE}" =~ ^packet.*$|^equinix.*$ ]]; then
@@ -113,6 +123,10 @@ while [[ -z "${selected_capability}" ]]; do
             selected_capability=""
         fi
         ;;
+    "Ingress")
+        echo "WARNING: Ingress should be always enabled, could not be disabled!"
+        selected_capability=""
+        ;;
     *)
         enabled_capabilities=${enabled_capabilities/${selected_capability}}
         for key in "${!dependency_caps[@]}"; do

ci-operator/step-registry/ipi/conf/capability/baselinecaps/vset/ipi-conf-capability-baselinecaps-vset-commands.sh

@@ -51,19 +51,21 @@ v411_set="vCurrent v4.11"
 v412_set="${v411_set} v4.12"
 v413_set="${v412_set} v4.13"
 v414_set="${v413_set} v4.14"
-# shellcheck disable=SC2034
 v415_set="${v414_set} v4.15"
-latest_version_set="v415_set"
+# shellcheck disable=SC2034
+v416_set="${v415_set} v4.16"
+latest_version_set="v416_set"
 
 # the content of each capset
 v411="baremetal marketplace openshift-samples"
 (( ocp_minor_version > 13 )) && v411="${v411} MachineAPI"
 v412=" ${v411} Console Insights Storage CSISnapshot"
 v413=" ${v412} NodeTuning"
 v414=" ${v413} Build DeploymentConfig ImageRegistry"
-# shellcheck disable=SC2034
 v415=" ${v414} OperatorLifecycleManager CloudCredential"
-latest_version="v415"
+# shellcheck disable=SC2034
+v416=" ${v415} CloudControllerManager Ingress"
+latest_version="v416"
 
 # define capability dependency
 declare -A dependency_caps

ci-operator/step-registry/ipi/conf/manual-creds/remove-unnecessary-creds/ipi-conf-manual-creds-remove-unnecessary-creds-commands.sh

@@ -59,7 +59,8 @@ v412=" ${v411} Console Insights Storage CSISnapshot"
 v413=" ${v412} NodeTuning"
 v414=" ${v413} MachineAPI Build DeploymentConfig ImageRegistry"
 v415=" ${v414} OperatorLifecycleManager CloudCredential"
-latest_defined="v415"
+v416=" ${v415} CloudControllerManager Ingress"
+latest_defined="v416"
 always_default="${!latest_defined}"
 
 # Determine vCurrent
@@ -97,6 +98,9 @@ case ${BASELINE_CAPABILITY_SET} in
 "v4.15")
   enabled_operators="${v415}"
   ;;
+"v4.16")
+  enabled_operators="${v416}"
+  ;;
 "vCurrent")
   enabled_operators="${vCurrent}"
   ;;