diff --git a/ci-operator/config/openshift/installer/openshift-installer-master.yaml b/ci-operator/config/openshift/installer/openshift-installer-master.yaml index fca24d499814e..ed3288f45e0b6 100644 --- a/ci-operator/config/openshift/installer/openshift-installer-master.yaml +++ b/ci-operator/config/openshift/installer/openshift-installer-master.yaml @@ -202,3 +202,7 @@ tests: steps: cluster_profile: packet workflow: baremetalds-e2e +- as: e2e-vsphere-steps + steps: + cluster_profile: vsphere + workflow: origin-e2e-vsphere diff --git a/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml b/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml index 8bc931873bd33..3207921f349e2 100644 --- a/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml +++ b/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml @@ -2269,6 +2269,83 @@ presubmits: secret: secretName: sentry-dsn trigger: (?m)^/test( | .* )e2e-vsphere,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - master + cluster: api.ci + context: ci/prow/e2e-vsphere-steps + decorate: true + decoration_config: + skip_cloning: true + labels: + ci-operator.openshift.io/prowgen-controlled: "true" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-installer-master-e2e-vsphere-steps + optional: true + rerun_command: /test e2e-vsphere-steps + spec: + containers: + - args: + - --artifact-dir=$(ARTIFACTS) + - --give-pr-author-access-to-namespace=true + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --kubeconfig=/etc/apici/kubeconfig + - --lease-server-password-file=/etc/boskos/password + - --lease-server-username=ci + - --lease-server=https://boskos-ci.svc.ci.openshift.org + - --secret-dir=/usr/local/e2e-vsphere-steps-cluster-profile + - --sentry-dsn-path=/etc/sentry-dsn/ci-operator + - --target=e2e-vsphere-steps + command: + - ci-operator + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/apici + name: apici-ci-operator-credentials + readOnly: true + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /usr/local/e2e-vsphere-steps-cluster-profile + name: cluster-profile + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/sentry-dsn + name: sentry-dsn + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: apici-ci-operator-credentials + secret: + items: + - key: sa.ci-operator.apici.config + path: kubeconfig + secretName: apici-ci-operator-credentials + - name: boskos + secret: + items: + - key: password + path: password + secretName: boskos-credentials + - name: cluster-profile + projected: + sources: + - secret: + name: cluster-secrets-vsphere + - name: pull-secret + secret: + secretName: regcred + - name: sentry-dsn + secret: + secretName: sentry-dsn + trigger: (?m)^/test( | .* )e2e-vsphere-steps,?($|\s.*) - agent: kubernetes always_run: true branches: diff --git a/ci-operator/step-registry/ipi/conf/vsphere/OWNERS b/ci-operator/step-registry/ipi/conf/vsphere/OWNERS new file mode 100644 index 0000000000000..f63e8872d2713 --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/vsphere/OWNERS @@ -0,0 +1,4 @@ +approvers: +- abhinavdahiya +- jcpowermac +- patrickdillon diff --git a/ci-operator/step-registry/ipi/conf/vsphere/dns/OWNERS b/ci-operator/step-registry/ipi/conf/vsphere/dns/OWNERS new file mode 100644 index 0000000000000..f63e8872d2713 --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/vsphere/dns/OWNERS @@ -0,0 +1,4 @@ +approvers: +- abhinavdahiya +- jcpowermac +- patrickdillon diff --git a/ci-operator/step-registry/ipi/conf/vsphere/dns/ipi-conf-vsphere-dns-commands.sh b/ci-operator/step-registry/ipi/conf/vsphere/dns/ipi-conf-vsphere-dns-commands.sh new file mode 100644 index 0000000000000..326ec0982fe11 --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/vsphere/dns/ipi-conf-vsphere-dns-commands.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +echo "origin-ci-int-aws.dev.rhcloud.com" > "${SHARED_DIR}"/basedomain.txt + +cluster_profile=/var/run/secrets/ci.openshift.io/cluster-profile +cluster_name=${NAMESPACE}-${JOB_NAME_HASH} +base_domain=$(<"${SHARED_DIR}"/basedomain.txt) +cluster_domain="${cluster_name}.${base_domain}" + +export AWS_SHARED_CREDENTIALS_FILE=${cluster_profile}/.awscred + +# Load array created in setup-vips: +# 0: API +# 1: Ingress +# 2: DNS +declare -a vips +mapfile -t vips < "${SHARED_DIR}"/vips.txt + +hosted_zone_id="$(aws route53 list-hosted-zones-by-name \ + --dns-name "${base_domain}" \ + --query "HostedZones[? Config.PrivateZone != \`true\` && Name == \`${base_domain}.\`].Id" \ + --output text)" + + +echo "Creating DNS records..." +cat > "${SHARED_DIR}"/dns-create.json < "${SHARED_DIR}"/dns-delete.json <- + Uses VIPs in ${SHARED_DIR}/vips.txt to create route53 dns records. Outputs + ${SHARED_DIR}/basedomain.txt to ensure consistent basedomain in conf and + deprovision steps. Saves batch job to delete DNS records to + ${SHARED_DIR}/dns-delete.json for use in deprovisioning. diff --git a/ci-operator/step-registry/ipi/conf/vsphere/ipi-conf-vsphere-chain.yaml b/ci-operator/step-registry/ipi/conf/vsphere/ipi-conf-vsphere-chain.yaml new file mode 100644 index 0000000000000..f3fffef1eb56b --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/vsphere/ipi-conf-vsphere-chain.yaml @@ -0,0 +1,11 @@ +chain: + as: ipi-conf-vsphere + steps: + - ref: ipi-conf-vsphere-vips + - ref: ipi-conf-vsphere-dns + - ref: ipi-conf + - ref: ipi-conf-vsphere + documentation: >- + The vSphere IPI configure step chain generates prerequisites for installing + a cluster: virtual IP addresses, DNS records, and the install-config.yaml. + Resources are created based on the cluster profile and optional input files. diff --git a/ci-operator/step-registry/ipi/conf/vsphere/ipi-conf-vsphere-commands.sh b/ci-operator/step-registry/ipi/conf/vsphere/ipi-conf-vsphere-commands.sh new file mode 100755 index 0000000000000..910af1e9fc94f --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/vsphere/ipi-conf-vsphere-commands.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +CONFIG="${SHARED_DIR}/install-config.yaml" +TFVARS_PATH=/var/run/secrets/ci.openshift.io/cluster-profile/secret.auto.tfvars +vsphere_user=$(grep -oP 'vsphere_user="\K[^"]+' ${TFVARS_PATH}) +vsphere_password=$(grep -oP 'vsphere_password="\K[^"]+' ${TFVARS_PATH}) +base_domain=$(<"${SHARED_DIR}"/basedomain.txt) + +declare -a vips +mapfile -t vips < "${SHARED_DIR}/vips.txt" + +cat >> "${CONFIG}" << EOF +baseDomain: $base_domain +platform: + vsphere: + cluster: devel + datacenter: dc1 + defaultDatastore: nvme-ds1 + network: VM Network + password: ${vsphere_password} + username: ${vsphere_user} + vCenter: vcsa-ci.vmware.devcluster.openshift.com + apiVIP: "${vips[0]}" + ingressVIP: "${vips[1]}" + dnsVIP: "${vips[2]}" +EOF diff --git a/ci-operator/step-registry/ipi/conf/vsphere/ipi-conf-vsphere-ref.yaml b/ci-operator/step-registry/ipi/conf/vsphere/ipi-conf-vsphere-ref.yaml new file mode 100644 index 0000000000000..87cb7497d69e9 --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/vsphere/ipi-conf-vsphere-ref.yaml @@ -0,0 +1,12 @@ +ref: + as: ipi-conf-vsphere + from: base + commands: ipi-conf-vsphere-commands.sh + resources: + requests: + cpu: 10m + memory: 100Mi + documentation: >- + The IPI vSphere configure step generates the vSphere-specific + install-config.yaml contents based on the cluster profile, + $SHARED_DIR/vips.txt, $SHARED_DIR/basedomain.txt, and optional input files. diff --git a/ci-operator/step-registry/ipi/conf/vsphere/vips/OWNERS b/ci-operator/step-registry/ipi/conf/vsphere/vips/OWNERS new file mode 100644 index 0000000000000..f63e8872d2713 --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/vsphere/vips/OWNERS @@ -0,0 +1,4 @@ +approvers: +- abhinavdahiya +- jcpowermac +- patrickdillon diff --git a/ci-operator/step-registry/ipi/conf/vsphere/vips/ipi-conf-vsphere-vips-commands.sh b/ci-operator/step-registry/ipi/conf/vsphere/vips/ipi-conf-vsphere-vips-commands.sh new file mode 100755 index 0000000000000..df037129de83f --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/vsphere/vips/ipi-conf-vsphere-vips-commands.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +tfvars_path=/var/run/secrets/ci.openshift.io/cluster-profile/secret.auto.tfvars +cluster_name=${NAMESPACE}-${JOB_NAME_HASH} +ipam_token=$(grep -oP 'ipam_token="\K[^"]+' ${tfvars_path}) + +# Array to hold virtual ips: +# 0: API +# 1: Ingress +# 2: DNS +declare -a vips + +echo "Reserving virtual ip addresses from the IPAM server..." +for i in {0..2} +do + args=$(jq -n \ + --arg hostn "$cluster_name-$i" \ + --arg token "$ipam_token" \ + '{network: "139.178.87.128", hostname: $hostn, ipam: "139.178.89.254", ipam_token: $token}') + + vip_json=$(echo "$args" | bash <(curl -s https://raw.githubusercontent.com/openshift/installer/master/upi/vsphere/machine/cidr_to_ip.sh)) + vips[$i]=$(echo "$vip_json" | jq -r .ip_address ) + if [[ -z ${vips[$i]} ]]; then + echo "error: Unable to reserve virtual IP address, exiting" 1>&2 + exit 1 + fi + echo "${vips[$i]}" >> "${SHARED_DIR}"/vips.txt +done + +echo "Reserved the following IP addresses..." +cat "${SHARED_DIR}"/vips.txt diff --git a/ci-operator/step-registry/ipi/conf/vsphere/vips/ipi-conf-vsphere-vips-ref.yaml b/ci-operator/step-registry/ipi/conf/vsphere/vips/ipi-conf-vsphere-vips-ref.yaml new file mode 100644 index 0000000000000..05ce7d4b9e892 --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/vsphere/vips/ipi-conf-vsphere-vips-ref.yaml @@ -0,0 +1,13 @@ +ref: + as: ipi-conf-vsphere-vips + from: upi-installer + commands: ipi-conf-vsphere-vips-commands.sh + resources: + requests: + cpu: 10m + memory: 100Mi + documentation: >- + Using secrets from the vSphere cluster profile, the vSphere VIP setup step + reserves IP addresses through IPAM and saves them to $SHARED_DIR/vips.txt + for later use in creating DNS records and the install config. They are also + needed in deprovisioning to release the reserved VIPs. diff --git a/ci-operator/step-registry/ipi/deprovision/vsphere/OWNERS b/ci-operator/step-registry/ipi/deprovision/vsphere/OWNERS new file mode 100644 index 0000000000000..f63e8872d2713 --- /dev/null +++ b/ci-operator/step-registry/ipi/deprovision/vsphere/OWNERS @@ -0,0 +1,4 @@ +approvers: +- abhinavdahiya +- jcpowermac +- patrickdillon diff --git a/ci-operator/step-registry/ipi/deprovision/vsphere/ipi-deprovision-vsphere-chain.yaml b/ci-operator/step-registry/ipi/deprovision/vsphere/ipi-deprovision-vsphere-chain.yaml new file mode 100644 index 0000000000000..2f2017e07513a --- /dev/null +++ b/ci-operator/step-registry/ipi/deprovision/vsphere/ipi-deprovision-vsphere-chain.yaml @@ -0,0 +1,7 @@ +chain: + as: ipi-deprovision-vsphere + steps: + - chain: gather + - ref: ipi-deprovision-vsphere + documentation: |- + The IPI deprovision step chain contains all the individual steps necessary to deprovision an OpenShift cluster. diff --git a/ci-operator/step-registry/ipi/deprovision/vsphere/ipi-deprovision-vsphere-commands.sh b/ci-operator/step-registry/ipi/deprovision/vsphere/ipi-deprovision-vsphere-commands.sh new file mode 100644 index 0000000000000..5fbb1127bb4e5 --- /dev/null +++ b/ci-operator/step-registry/ipi/deprovision/vsphere/ipi-deprovision-vsphere-commands.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +cluster_profile=/var/run/secrets/ci.openshift.io/cluster-profile +tfvars_path=/var/run/secrets/ci.openshift.io/cluster-profile/secret.auto.tfvars +base_domain=$(<"${SHARED_DIR}"/basedomain.txt) +cluster_name=${NAMESPACE}-${JOB_NAME_HASH} +ipam_token=$(grep -oP 'ipam_token="\K[^"]+' ${tfvars_path}) + +export AWS_SHARED_CREDENTIALS_FILE=${cluster_profile}/.awscred + +echo "Deprovisioning cluster ..." +cp -ar "${SHARED_DIR}" /tmp/installer +TF_LOG=debug openshift-install --dir /tmp/installer destroy cluster +cp /tmp/installer/.openshift_install.log "${ARTIFACT_DIR}/" + +hosted_zone_id="$(aws route53 list-hosted-zones-by-name \ + --dns-name "${base_domain}" \ + --query "HostedZones[? Config.PrivateZone != \`true\` && Name == \`${base_domain}.\`].Id" \ + --output text)" + +echo "Releasing IP addresses from IPAM server..." +for i in {0..2} +do + curl -s "http://139.178.89.254/api/removeHost.php?apiapp=address&apitoken=${ipam_token}&host=${cluster_name}-$i" +done + +echo "Deleting Route53 DNS records..." +aws route53 change-resource-record-sets --hosted-zone-id "$hosted_zone_id" --change-batch file:///"${SHARED_DIR}"/dns-delete.json diff --git a/ci-operator/step-registry/ipi/deprovision/vsphere/ipi-deprovision-vsphere-ref.yaml b/ci-operator/step-registry/ipi/deprovision/vsphere/ipi-deprovision-vsphere-ref.yaml new file mode 100644 index 0000000000000..1d3d5b797ba38 --- /dev/null +++ b/ci-operator/step-registry/ipi/deprovision/vsphere/ipi-deprovision-vsphere-ref.yaml @@ -0,0 +1,12 @@ +ref: + as: ipi-deprovision-vsphere + from: upi-installer + commands: ipi-deprovision-vsphere-commands.sh + resources: + requests: + cpu: 1000m + memory: 100Mi + documentation: >- + Reads the VIP and DNS records created in the setup phases from $SHARED_DIR + and delete them. Also uses $SHARED_DIR/basedomain.txt to ensure consistent + basedomain as setup steps. diff --git a/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh b/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh index 861e4161fb85b..30dbc9760a127 100755 --- a/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh +++ b/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh @@ -17,6 +17,7 @@ case "${CLUSTER_TYPE}" in aws) export AWS_SHARED_CREDENTIALS_FILE=${cluster_profile}/.awscred;; azure4) export AZURE_AUTH_LOCATION=${cluster_profile}/osServicePrincipal.json;; gcp) export GOOGLE_CLOUD_KEYFILE_JSON=${cluster_profile}/gce.json;; +vsphere) ;; *) echo >&2 "Unsupported cluster type '${CLUSTER_TYPE}'" esac diff --git a/ci-operator/step-registry/ipi/install/vsphere/OWNERS b/ci-operator/step-registry/ipi/install/vsphere/OWNERS new file mode 100644 index 0000000000000..f63e8872d2713 --- /dev/null +++ b/ci-operator/step-registry/ipi/install/vsphere/OWNERS @@ -0,0 +1,4 @@ +approvers: +- abhinavdahiya +- jcpowermac +- patrickdillon diff --git a/ci-operator/step-registry/ipi/install/vsphere/ipi-install-vsphere-chain.yaml b/ci-operator/step-registry/ipi/install/vsphere/ipi-install-vsphere-chain.yaml new file mode 100644 index 0000000000000..16bc97790b1c1 --- /dev/null +++ b/ci-operator/step-registry/ipi/install/vsphere/ipi-install-vsphere-chain.yaml @@ -0,0 +1,7 @@ +chain: + as: ipi-install-vsphere + steps: + - ref: ipi-install-rbac + - ref: ipi-install-vsphere + documentation: |- + The IPI install step chain contains all the individual steps necessary to install an OpenShift cluster. \ No newline at end of file diff --git a/ci-operator/step-registry/ipi/install/vsphere/ipi-install-vsphere-commands.sh b/ci-operator/step-registry/ipi/install/vsphere/ipi-install-vsphere-commands.sh new file mode 120000 index 0000000000000..b27c3a0d97c88 --- /dev/null +++ b/ci-operator/step-registry/ipi/install/vsphere/ipi-install-vsphere-commands.sh @@ -0,0 +1 @@ +../install/ipi-install-install-commands.sh \ No newline at end of file diff --git a/ci-operator/step-registry/ipi/install/vsphere/ipi-install-vsphere-ref.yaml b/ci-operator/step-registry/ipi/install/vsphere/ipi-install-vsphere-ref.yaml new file mode 100644 index 0000000000000..c8c8c190c3482 --- /dev/null +++ b/ci-operator/step-registry/ipi/install/vsphere/ipi-install-vsphere-ref.yaml @@ -0,0 +1,12 @@ +ref: + as: ipi-install-vsphere + from: upi-installer + commands: ipi-install-vsphere-commands.sh + resources: + requests: + cpu: 1000m + memory: 2Gi + documentation: >- + The vSphere IPI install step runs the OpenShift Installer in order to bring + up an OpenShift cluster. Unlike the other IPI installs, the installer image + must include vCenter CA certs. diff --git a/ci-operator/step-registry/origin/e2e/vsphere/OWNERS b/ci-operator/step-registry/origin/e2e/vsphere/OWNERS new file mode 100644 index 0000000000000..f63e8872d2713 --- /dev/null +++ b/ci-operator/step-registry/origin/e2e/vsphere/OWNERS @@ -0,0 +1,4 @@ +approvers: +- abhinavdahiya +- jcpowermac +- patrickdillon diff --git a/ci-operator/step-registry/origin/e2e/vsphere/origin-e2e-vsphere-workflow.yaml b/ci-operator/step-registry/origin/e2e/vsphere/origin-e2e-vsphere-workflow.yaml new file mode 100644 index 0000000000000..beb07ea3b3410 --- /dev/null +++ b/ci-operator/step-registry/origin/e2e/vsphere/origin-e2e-vsphere-workflow.yaml @@ -0,0 +1,12 @@ +workflow: + as: origin-e2e-vsphere + steps: + pre: + - chain: ipi-conf-vsphere + - chain: ipi-install-vsphere + test: + - ref: origin-e2e-test + post: + - chain: ipi-deprovision-vsphere + documentation: |- + VSphere E2E wraps the origin E2E workflow and executes the common end-to-end test suite.