From 7912bb17cb0acd85379265c8c28068647e53d15e Mon Sep 17 00:00:00 2001 From: Mike Gahagan Date: Mon, 7 Oct 2024 13:52:34 -0400 Subject: [PATCH 1/5] create aro cluster with pre-existing network security group --- .../step-registry/aro/provision/nsg/OWNERS | 1 + .../nsg/aro-provision-nsg-commands.sh | 45 +++++++++++++++++++ .../nsg/aro-provision-nsg-ref.metadata.json | 17 +++++++ .../provision/nsg/aro-provision-nsg-ref.yaml | 17 +++++++ .../rehearse/azure/aro/byonsg/OWNERS | 1 + ...se-azure-aro-byonsg-workflow.metadata.json | 17 +++++++ ...er-rehearse-azure-aro-byonsg-workflow.yaml | 17 +++++++ 7 files changed, 115 insertions(+) create mode 120000 ci-operator/step-registry/aro/provision/nsg/OWNERS create mode 100644 ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-commands.sh create mode 100644 ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-ref.metadata.json create mode 100644 ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-ref.yaml create mode 120000 ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/OWNERS create mode 100644 ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/cucushift-installer-rehearse-azure-aro-byonsg-workflow.metadata.json create mode 100644 ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/cucushift-installer-rehearse-azure-aro-byonsg-workflow.yaml diff --git a/ci-operator/step-registry/aro/provision/nsg/OWNERS b/ci-operator/step-registry/aro/provision/nsg/OWNERS new file mode 120000 index 000000000000..ec405d65a79d --- /dev/null +++ b/ci-operator/step-registry/aro/provision/nsg/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-commands.sh b/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-commands.sh new file mode 100644 index 000000000000..67326ff3e440 --- /dev/null +++ b/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-commands.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +#echo "Installing oc binary" +#curl -s https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/openshift-client-linux.tar.gz | tar zxvf - oc +#chmod +x oc +CLUSTER=${CLUSTER:="${NAMESPACE}-${UNIQUE_HASH}"} +RESOURCEGROUP=${RESOURCEGROUP:=$(cat "${SHARED_DIR}/resourcegroup")} +VNET=${VNET:=$(cat "${SHARED_DIR}"/vnet)} +LOCATION=${LOCATION:=${LEASED_RESOURCE}} +AZURE_AUTH_LOCATION="${CLUSTER_PROFILE_DIR}/osServicePrincipal.json" +AZURE_AUTH_CLIENT_ID="$(<"${AZURE_AUTH_LOCATION}" jq -r .clientId)" +AZURE_AUTH_CLIENT_SECRET="$(<"${AZURE_AUTH_LOCATION}" jq -r .clientSecret)" +AZURE_AUTH_TENANT_ID="$(<"${AZURE_AUTH_LOCATION}" jq -r .tenantId)" +MASTER_SUBNET_NAME=${MASTER_SUBNET_NAME:=$(grep controlPlaneSubnet ${SHARED_DIR}/customer_vnet_subnets.yaml | cut -d ":" -f 2 | tr -d [:blank:])} +WORKER_SUBNET_NAME=${WORKER_SUBNET_NAME:=$(grep computeSubnet ${SHARED_DIR}/customer_vnet_subnets.yaml | cut -d ":" -f 2 | tr -d [:blank:])} +NSG=${NSG:=${CLUSTER}-nsg} +NSG_OPEN_PORTS=${NSG_OPEN_PORTS:="80 443 6443"} + +# get az-cli, do feature adds for cloud if needed +# +echo "Logging into Azure Cloud" +# log in with az +if [[ "${CLUSTER_TYPE}" == "azuremag" ]]; then + az cloud set --name AzureUSGovernment +else + az cloud set --name AzureCloud +fi +az login --service-principal -u "${AZURE_AUTH_CLIENT_ID}" -p "${AZURE_AUTH_CLIENT_SECRET}" --tenant "${AZURE_AUTH_TENANT_ID}" --output none + +echo "Creating nsg: ${NSG} in resource group ${RESOURCEGROUP} in location: ${LOCATION}" +# see https://raw.githubusercontent.com/openshift/osde2e/main/ci/create-aro-cluster.sh +# create the resourcegroup to contain the cluster object and vnet +az network nsg create -g "${RESOURCEGROUP}" -n "${NSG}" +az network nsg rule create -g "${RESOURCEGROUP}" --nsg-name "${NSG}" -n "${NSG}-allow" --priority 1000 --access Allow --source-port-ranges "*" --destination-port-ranges "${NSG_OPEN_PORTS}" +echo "Updating ${MASTER_SUBNET_NAME} in vnet ${VNET}, attaching ${NSG}" +az network vnet subnet update -g "${RESOURCEGROUP}" -n "${MASTER_SUBNET_NAME}" --vnet-name "${VNET}" --network-security-group "${NSG}" +echo "Updating ${WORKER_SUBNET_NAME} in vnet ${VNET}, attaching ${NSG}" +az network vnet subnet update -g "${RESOURCEGROUP}" -n "${WORKER_SUBNET_NAME}" --vnet-name "${VNET}" --network-security-group "${NSG}" + + + diff --git a/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-ref.metadata.json b/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-ref.metadata.json new file mode 100644 index 000000000000..eaaa516d9dcf --- /dev/null +++ b/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-ref.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "aro/provision/nsg/aro-provision-nsg-ref.yaml", + "owners": { + "approvers": [ + "jianlinliu", + "yunjiang29", + "mgahagan73", + "MayXuQQ" + ], + "reviewers": [ + "jianlinliu", + "yunjiang29", + "mgahagan73", + "MayXuQQ" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-ref.yaml b/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-ref.yaml new file mode 100644 index 000000000000..93c9e47ddfd2 --- /dev/null +++ b/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-ref.yaml @@ -0,0 +1,17 @@ +ref: + as: aro-provision-nsg + from_image: + namespace: ocp + name: "4.14" + tag: upi-installer + commands: aro-provision-nsg-commands.sh + resources: + requests: + cpu: 10m + memory: 100Mi + env: + - name: NSG_OPEN_PORTS + default: "80 443 6443" + documentation: "Space separated list of ports to open for the network security group" + documentation: |- + Provision a custom network security group for an aro cluster. \ No newline at end of file diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/OWNERS b/ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/OWNERS new file mode 120000 index 000000000000..ec405d65a79d --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/cucushift-installer-rehearse-azure-aro-byonsg-workflow.metadata.json b/ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/cucushift-installer-rehearse-azure-aro-byonsg-workflow.metadata.json new file mode 100644 index 000000000000..286d85be39b4 --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/cucushift-installer-rehearse-azure-aro-byonsg-workflow.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "cucushift/installer/rehearse/azure/aro/byonsg/cucushift-installer-rehearse-azure-aro-byonsg-workflow.yaml", + "owners": { + "approvers": [ + "jianlinliu", + "yunjiang29", + "jinyunma", + "mgahagan73" + ], + "reviewers": [ + "jianlinliu", + "yunjiang29", + "jinyunma", + "mgahagan73" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/cucushift-installer-rehearse-azure-aro-byonsg-workflow.yaml b/ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/cucushift-installer-rehearse-azure-aro-byonsg-workflow.yaml new file mode 100644 index 000000000000..a25ea2148362 --- /dev/null +++ b/ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byonsg/cucushift-installer-rehearse-azure-aro-byonsg-workflow.yaml @@ -0,0 +1,17 @@ +workflow: + as: cucushift-installer-rehearse-azure-aro-byonsg + steps: + pre: + - ref: azure-provision-resourcegroup + - ref: aro-provision-vnet + - ref: aro-provision-nsg + - ref: aro-provision-cluster + - ref: ipi-install-rbac + - ref: openshift-cluster-bot-rbac + - ref: enable-qe-catalogsource + post: + - ref: aro-deprovision + env: + ARO_BYO_NSG: "true" + documentation: |- + This is the workflow to trigger Prow's rehearsal test when submitting installer steps/chain/workflow for aro From 0b402e901163f2890bc88097324edc9222f77eb6 Mon Sep 17 00:00:00 2001 From: Mike Gahagan Date: Mon, 7 Oct 2024 13:54:36 -0400 Subject: [PATCH 2/5] add nsg to provisoner --- .../aro/provision/cluster/aro-provision-cluster-commands.sh | 6 ++++++ .../aro/provision/cluster/aro-provision-cluster-ref.yaml | 3 +++ 2 files changed, 9 insertions(+) diff --git a/ci-operator/step-registry/aro/provision/cluster/aro-provision-cluster-commands.sh b/ci-operator/step-registry/aro/provision/cluster/aro-provision-cluster-commands.sh index 4b347a0526ab..1157395bf449 100644 --- a/ci-operator/step-registry/aro/provision/cluster/aro-provision-cluster-commands.sh +++ b/ci-operator/step-registry/aro/provision/cluster/aro-provision-cluster-commands.sh @@ -22,6 +22,7 @@ ARO_INGRESS_VISIBILITY=${ARO_INGRESS_VISIBILITY:=""} ARO_API_SERVER_VISIBILITY=${ARO_API_SERVER_VISIBILITY:=""} ARO_OUTBOUND_TYPE=${ARO_OUTBOUND_TYPE:=""} ARO_FIPS=${ARO_FIPS:="false"} +ARO_BYO_NSG=${ARO_BYO_NSG:="false"} echo $CLUSTER > $SHARED_DIR/cluster-name echo $LOCATION > $SHARED_DIR/location @@ -101,6 +102,11 @@ if [[ ${ARO_FIPS} == "true" ]]; then CREATE_CMD="${CREATE_CMD} --fips ${ARO_FIPS}" fi +# BYO NSG support +if [[ ${ARO_BYO_NSG} == "true" ]]; then + CREATE_CMD="${CREATE_CMD} --enable-preconfigured-nsg" +fi + echo "Running ARO create command:" echo "${CREATE_CMD}" eval "${CREATE_CMD}" > ${SHARED_DIR}/clusterinfo diff --git a/ci-operator/step-registry/aro/provision/cluster/aro-provision-cluster-ref.yaml b/ci-operator/step-registry/aro/provision/cluster/aro-provision-cluster-ref.yaml index 1729e2f7754c..b1b0de896b90 100644 --- a/ci-operator/step-registry/aro/provision/cluster/aro-provision-cluster-ref.yaml +++ b/ci-operator/step-registry/aro/provision/cluster/aro-provision-cluster-ref.yaml @@ -34,5 +34,8 @@ ref: - name: ARO_FIPS default: "false" documentation: "Use FIPS validated cryptography modules. Allowed values: false, true." + - name: ARO_BYO_NSG + default: "false" + documentation: "ARO cluster will use an existing network security group. The NSG must exist and be attached to the subnets before creating cluster. Allowed values: false, true." documentation: |- Provision an aro cluster. \ No newline at end of file From bc52b09317c726fda8f8b6850ffaef8e147f58f6 Mon Sep 17 00:00:00 2001 From: Mike Gahagan Date: Mon, 7 Oct 2024 14:02:20 -0400 Subject: [PATCH 3/5] fix quoting --- .../aro/provision/nsg/aro-provision-nsg-commands.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-commands.sh b/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-commands.sh index 67326ff3e440..badf22188516 100644 --- a/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-commands.sh +++ b/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-commands.sh @@ -15,8 +15,8 @@ AZURE_AUTH_LOCATION="${CLUSTER_PROFILE_DIR}/osServicePrincipal.json" AZURE_AUTH_CLIENT_ID="$(<"${AZURE_AUTH_LOCATION}" jq -r .clientId)" AZURE_AUTH_CLIENT_SECRET="$(<"${AZURE_AUTH_LOCATION}" jq -r .clientSecret)" AZURE_AUTH_TENANT_ID="$(<"${AZURE_AUTH_LOCATION}" jq -r .tenantId)" -MASTER_SUBNET_NAME=${MASTER_SUBNET_NAME:=$(grep controlPlaneSubnet ${SHARED_DIR}/customer_vnet_subnets.yaml | cut -d ":" -f 2 | tr -d [:blank:])} -WORKER_SUBNET_NAME=${WORKER_SUBNET_NAME:=$(grep computeSubnet ${SHARED_DIR}/customer_vnet_subnets.yaml | cut -d ":" -f 2 | tr -d [:blank:])} +MASTER_SUBNET_NAME=${MASTER_SUBNET_NAME:=$(grep controlPlaneSubnet ${SHARED_DIR}/customer_vnet_subnets.yaml | cut -d ":" -f 2 | tr -d "[:blank:]")} +WORKER_SUBNET_NAME=${WORKER_SUBNET_NAME:=$(grep computeSubnet ${SHARED_DIR}/customer_vnet_subnets.yaml | cut -d ":" -f 2 | tr -d "[:blank:]")} NSG=${NSG:=${CLUSTER}-nsg} NSG_OPEN_PORTS=${NSG_OPEN_PORTS:="80 443 6443"} From 2aa5ae8490dccc8d1673dc83707477385e8d673b Mon Sep 17 00:00:00 2001 From: Mike Gahagan Date: Mon, 7 Oct 2024 14:24:03 -0400 Subject: [PATCH 4/5] openshift-verification-tests-master__installer-rehearse-4.15.yaml --- ...hift-verification-tests-master__installer-rehearse-4.15.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-operator/config/openshift/verification-tests/openshift-verification-tests-master__installer-rehearse-4.15.yaml b/ci-operator/config/openshift/verification-tests/openshift-verification-tests-master__installer-rehearse-4.15.yaml index 6101d81602ca..2a31dc6088cf 100644 --- a/ci-operator/config/openshift/verification-tests/openshift-verification-tests-master__installer-rehearse-4.15.yaml +++ b/ci-operator/config/openshift/verification-tests/openshift-verification-tests-master__installer-rehearse-4.15.yaml @@ -67,7 +67,7 @@ tests: env: ARO_CLUSTER_VERSION: 4.15.27 ARO_FIPS: "true" - workflow: cucushift-installer-rehearse-azure-aro-private + workflow: cucushift-installer-rehearse-azure-aro-byonsg - as: installer-rehearse-ibmcloud cron: '@yearly' steps: From 9dd4d0df750a9a4594e3b2b441edc1a166c631a7 Mon Sep 17 00:00:00 2001 From: Mike Gahagan Date: Mon, 7 Oct 2024 14:57:05 -0400 Subject: [PATCH 5/5] try without quotes --- .../aro/provision/nsg/aro-provision-nsg-commands.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-commands.sh b/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-commands.sh index badf22188516..f5e969461300 100644 --- a/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-commands.sh +++ b/ci-operator/step-registry/aro/provision/nsg/aro-provision-nsg-commands.sh @@ -35,7 +35,7 @@ echo "Creating nsg: ${NSG} in resource group ${RESOURCEGROUP} in location: ${LOC # see https://raw.githubusercontent.com/openshift/osde2e/main/ci/create-aro-cluster.sh # create the resourcegroup to contain the cluster object and vnet az network nsg create -g "${RESOURCEGROUP}" -n "${NSG}" -az network nsg rule create -g "${RESOURCEGROUP}" --nsg-name "${NSG}" -n "${NSG}-allow" --priority 1000 --access Allow --source-port-ranges "*" --destination-port-ranges "${NSG_OPEN_PORTS}" +az network nsg rule create -g "${RESOURCEGROUP}" --nsg-name "${NSG}" -n "${NSG}-allow" --priority 1000 --access Allow --source-port-ranges "*" --destination-port-ranges ${NSG_OPEN_PORTS} echo "Updating ${MASTER_SUBNET_NAME} in vnet ${VNET}, attaching ${NSG}" az network vnet subnet update -g "${RESOURCEGROUP}" -n "${MASTER_SUBNET_NAME}" --vnet-name "${VNET}" --network-security-group "${NSG}" echo "Updating ${WORKER_SUBNET_NAME} in vnet ${VNET}, attaching ${NSG}"