Create ARO cluster with pre-existing network security group.

ci-operator/config/openshift/verification-tests/openshift-verification-tests-master__installer-rehearse-4.15.yaml

@@ -67,7 +67,7 @@ tests:
     env:
       ARO_CLUSTER_VERSION: 4.15.27
       ARO_FIPS: "true"
-    workflow: cucushift-installer-rehearse-azure-aro-private
+    workflow: cucushift-installer-rehearse-azure-aro-byonsg
 - as: installer-rehearse-ibmcloud
   cron: '@yearly'
   steps:

ci-operator/step-registry/aro/provision/cluster/aro-provision-cluster-commands.sh

@@ -22,6 +22,7 @@ ARO_INGRESS_VISIBILITY=${ARO_INGRESS_VISIBILITY:=""}
 ARO_API_SERVER_VISIBILITY=${ARO_API_SERVER_VISIBILITY:=""}
 ARO_OUTBOUND_TYPE=${ARO_OUTBOUND_TYPE:=""}
 ARO_FIPS=${ARO_FIPS:="false"}
+ARO_BYO_NSG=${ARO_BYO_NSG:="false"}
 
 echo $CLUSTER > $SHARED_DIR/cluster-name
 echo $LOCATION > $SHARED_DIR/location
@@ -101,6 +102,11 @@ if [[ ${ARO_FIPS} == "true" ]]; then
   CREATE_CMD="${CREATE_CMD} --fips ${ARO_FIPS}"
 fi
 
+# BYO NSG support
+if [[ ${ARO_BYO_NSG} == "true" ]]; then
+  CREATE_CMD="${CREATE_CMD} --enable-preconfigured-nsg"
+fi
+
 echo "Running ARO create command:"
 echo "${CREATE_CMD}"
 eval "${CREATE_CMD}" > ${SHARED_DIR}/clusterinfo

ci-operator/step-registry/aro/provision/cluster/aro-provision-cluster-ref.yaml

@@ -34,5 +34,8 @@ ref:
   - name: ARO_FIPS
     default: "false"
     documentation: "Use FIPS validated cryptography modules.  Allowed values: false, true."
+  - name: ARO_BYO_NSG
+    default: "false"
+    documentation: "ARO cluster will use an existing network security group. The NSG must exist and be attached to the subnets before creating cluster. Allowed values: false, true."
   documentation: |-
     Provision an aro cluster.

ci-operator/step-registry/aro/provision/nsg/OWNERS

@@ -0,0 +1 @@
+../OWNERS

ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-commands.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+#echo "Installing oc binary"
+#curl -s https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/openshift-client-linux.tar.gz | tar zxvf - oc
+#chmod +x oc
+CLUSTER=${CLUSTER:="${NAMESPACE}-${UNIQUE_HASH}"}
+RESOURCEGROUP=${RESOURCEGROUP:=$(cat "${SHARED_DIR}/resourcegroup")}
+VNET=${VNET:=$(cat "${SHARED_DIR}"/vnet)}
+LOCATION=${LOCATION:=${LEASED_RESOURCE}}
+AZURE_AUTH_LOCATION="${CLUSTER_PROFILE_DIR}/osServicePrincipal.json"
+AZURE_AUTH_CLIENT_ID="$(<"${AZURE_AUTH_LOCATION}" jq -r .clientId)"
+AZURE_AUTH_CLIENT_SECRET="$(<"${AZURE_AUTH_LOCATION}" jq -r .clientSecret)"
+AZURE_AUTH_TENANT_ID="$(<"${AZURE_AUTH_LOCATION}" jq -r .tenantId)"
+MASTER_SUBNET_NAME=${MASTER_SUBNET_NAME:=$(grep controlPlaneSubnet ${SHARED_DIR}/customer_vnet_subnets.yaml | cut -d ":" -f 2 | tr -d "[:blank:]")}
+WORKER_SUBNET_NAME=${WORKER_SUBNET_NAME:=$(grep computeSubnet ${SHARED_DIR}/customer_vnet_subnets.yaml | cut -d ":" -f 2 | tr -d "[:blank:]")}
+NSG=${NSG:=${CLUSTER}-nsg}
+NSG_OPEN_PORTS=${NSG_OPEN_PORTS:="80 443 6443"}
+
+# get az-cli, do feature adds for cloud if needed
+# 
+echo "Logging into Azure Cloud"
+# log in with az
+if [[ "${CLUSTER_TYPE}" == "azuremag" ]]; then
+    az cloud set --name AzureUSGovernment
+else
+    az cloud set --name AzureCloud
+fi
+az login --service-principal -u "${AZURE_AUTH_CLIENT_ID}" -p "${AZURE_AUTH_CLIENT_SECRET}" --tenant "${AZURE_AUTH_TENANT_ID}" --output none
+
+echo "Creating nsg: ${NSG} in resource group ${RESOURCEGROUP} in location: ${LOCATION}"
+# see https://raw.githubusercontent.com/openshift/osde2e/main/ci/create-aro-cluster.sh
+# create the resourcegroup to contain the cluster object and vnet
+az network nsg create -g "${RESOURCEGROUP}" -n "${NSG}"
+az network nsg rule create -g "${RESOURCEGROUP}" --nsg-name "${NSG}" -n "${NSG}-allow" --priority 1000 --access Allow --source-port-ranges "*" --destination-port-ranges ${NSG_OPEN_PORTS}
+echo "Updating ${MASTER_SUBNET_NAME} in vnet ${VNET}, attaching ${NSG}"
+az network vnet subnet update -g "${RESOURCEGROUP}" -n "${MASTER_SUBNET_NAME}" --vnet-name "${VNET}" --network-security-group "${NSG}"
+echo "Updating ${WORKER_SUBNET_NAME} in vnet ${VNET}, attaching ${NSG}"
+az network vnet subnet update -g "${RESOURCEGROUP}" -n "${WORKER_SUBNET_NAME}" --vnet-name "${VNET}" --network-security-group "${NSG}"
+
+
+

ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-ref.metadata.json

@@ -0,0 +1,17 @@
+{
+	"path": "aro/provision/nsg/aro-provision-nsg-ref.yaml",
+	"owners": {
+		"approvers": [
+			"jianlinliu",
+			"yunjiang29",
+			"mgahagan73",
+			"MayXuQQ"
+		],
+		"reviewers": [
+			"jianlinliu",
+			"yunjiang29",
+			"mgahagan73",
+			"MayXuQQ"
+		]
+	}
+}

ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-ref.yaml

@@ -0,0 +1,17 @@
+ref:
+  as: aro-provision-nsg
+  from_image:
+    namespace: ocp
+    name: "4.14"
+    tag: upi-installer
+  commands: aro-provision-nsg-commands.sh
+  resources:
+    requests:
+      cpu: 10m
+      memory: 100Mi
+  env:
+  - name: NSG_OPEN_PORTS
+    default: "80 443 6443"
+    documentation: "Space separated list of ports to open for the network security group"
+  documentation: |-
+    Provision a custom network security group for an aro cluster.

ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/OWNERS

@@ -0,0 +1 @@
+../OWNERS

ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/cucushift-installer-rehearse-azure-aro-byonsg-workflow.metadata.json

@@ -0,0 +1,17 @@
+{
+	"path": "cucushift/installer/rehearse/azure/aro/byonsg/cucushift-installer-rehearse-azure-aro-byonsg-workflow.yaml",
+	"owners": {
+		"approvers": [
+			"jianlinliu",
+			"yunjiang29",
+			"jinyunma",
+			"mgahagan73"
+		],
+		"reviewers": [
+			"jianlinliu",
+			"yunjiang29",
+			"jinyunma",
+			"mgahagan73"
+		]
+	}
+}

ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/cucushift-installer-rehearse-azure-aro-byonsg-workflow.yaml

@@ -0,0 +1,17 @@
+workflow:
+  as: cucushift-installer-rehearse-azure-aro-byonsg
+  steps: 
+    pre:
+    - ref: azure-provision-resourcegroup
+    - ref: aro-provision-vnet
+    - ref: aro-provision-nsg
+    - ref: aro-provision-cluster
+    - ref: ipi-install-rbac
+    - ref: openshift-cluster-bot-rbac
+    - ref: enable-qe-catalogsource
+    post:
+    - ref: aro-deprovision
+    env:
+      ARO_BYO_NSG: "true"
+  documentation: |-
+    This is the workflow to trigger Prow's rehearsal test when submitting installer steps/chain/workflow for aro