diff --git a/ci-operator/config/openshift/installer/openshift-installer-master.yaml b/ci-operator/config/openshift/installer/openshift-installer-master.yaml index 8d5f5ed4be0a2..b1d616f8ffb82 100644 --- a/ci-operator/config/openshift/installer/openshift-installer-master.yaml +++ b/ci-operator/config/openshift/installer/openshift-installer-master.yaml @@ -444,6 +444,14 @@ tests: test: - ref: openstack-noop workflow: openshift-e2e-openstack-ipi +- as: e2e-azurestack + steps: + cluster_profile: azurestack + workflow: openshift-e2e-azurestack +- as: e2e-azurestack-upi + steps: + cluster_profile: azurestack + workflow: openshift-e2e-azurestack-upi zz_generated_metadata: branch: master org: openshift diff --git a/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml b/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml index 8522e0f753189..0fa7febdb018c 100644 --- a/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml +++ b/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml @@ -1157,6 +1157,152 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-azure-upi,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^master$ + - ^master- + cluster: build01 + context: ci/prow/e2e-azurestack + decorate: true + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-installer-master-e2e-azurestack + optional: true + rerun_command: /test e2e-azurestack + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --secret-dir=/usr/local/e2e-azurestack-cluster-profile + - --target=e2e-azurestack + command: + - ci-operator + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /usr/local/e2e-azurestack-cluster-profile + name: cluster-profile + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: cluster-profile + projected: + sources: + - secret: + name: cluster-secrets-azurestack + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-azurestack,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^master$ + - ^master- + cluster: build01 + context: ci/prow/e2e-azurestack-upi + decorate: true + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-installer-master-e2e-azurestack-upi + optional: true + rerun_command: /test e2e-azurestack-upi + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --secret-dir=/usr/local/e2e-azurestack-upi-cluster-profile + - --target=e2e-azurestack-upi + command: + - ci-operator + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /usr/local/e2e-azurestack-upi-cluster-profile + name: cluster-profile + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: cluster-profile + projected: + sources: + - secret: + name: cluster-secrets-azurestack + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-azurestack-upi,?($|\s.*) - agent: kubernetes always_run: false branches: diff --git a/ci-operator/step-registry/ipi/azurestack/OWNERS b/ci-operator/step-registry/ipi/azurestack/OWNERS new file mode 100644 index 0000000000000..d3f5334072603 --- /dev/null +++ b/ci-operator/step-registry/ipi/azurestack/OWNERS @@ -0,0 +1,8 @@ +approvers: +- staebler +- rna-afk +- patrickdillon +- mtnbikenc +- e-tienne +- jhixson74 +- jstuever diff --git a/ci-operator/step-registry/ipi/azurestack/ipi-azurestack-workflow.metadata.json b/ci-operator/step-registry/ipi/azurestack/ipi-azurestack-workflow.metadata.json new file mode 100644 index 0000000000000..3a71ba310d610 --- /dev/null +++ b/ci-operator/step-registry/ipi/azurestack/ipi-azurestack-workflow.metadata.json @@ -0,0 +1,14 @@ +{ + "path": "ipi/azurestack/ipi-azurestack-workflow.yaml", + "owners": { + "approvers": [ + "staebler", + "rna-afk", + "patrickdillon", + "mtnbikenc", + "e-tienne", + "jhixson74", + "jstuever" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/ipi/azurestack/ipi-azurestack-workflow.yaml b/ci-operator/step-registry/ipi/azurestack/ipi-azurestack-workflow.yaml new file mode 100644 index 0000000000000..cc5cae3c9b286 --- /dev/null +++ b/ci-operator/step-registry/ipi/azurestack/ipi-azurestack-workflow.yaml @@ -0,0 +1,15 @@ +workflow: + as: ipi-azurestack + steps: + pre: + - chain: ipi-azurestack-pre + post: + - chain: ipi-azurestack-post + documentation: |- + The IPI workflow provides pre- and post- steps that provision and + deprovision an OpenShift cluster with a default configuration on Azure + Stack Hub allowing job authors to inject their own end-to-end test logic. + + All modifications to this workflow should be done by modifying the + `ipi-azurestack-{pre,post}` chains to allow other workflows to mimic and extend + this base workflow without a need to backport changes. diff --git a/ci-operator/step-registry/ipi/azurestack/post/OWNERS b/ci-operator/step-registry/ipi/azurestack/post/OWNERS new file mode 100644 index 0000000000000..d3f5334072603 --- /dev/null +++ b/ci-operator/step-registry/ipi/azurestack/post/OWNERS @@ -0,0 +1,8 @@ +approvers: +- staebler +- rna-afk +- patrickdillon +- mtnbikenc +- e-tienne +- jhixson74 +- jstuever diff --git a/ci-operator/step-registry/ipi/azurestack/post/ipi-azurestack-post-chain.metadata.json b/ci-operator/step-registry/ipi/azurestack/post/ipi-azurestack-post-chain.metadata.json new file mode 100644 index 0000000000000..690ed50392907 --- /dev/null +++ b/ci-operator/step-registry/ipi/azurestack/post/ipi-azurestack-post-chain.metadata.json @@ -0,0 +1,14 @@ +{ + "path": "ipi/azurestack/post/ipi-azurestack-post-chain.yaml", + "owners": { + "approvers": [ + "staebler", + "rna-afk", + "patrickdillon", + "mtnbikenc", + "e-tienne", + "jhixson74", + "jstuever" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/ipi/azurestack/post/ipi-azurestack-post-chain.yaml b/ci-operator/step-registry/ipi/azurestack/post/ipi-azurestack-post-chain.yaml new file mode 100644 index 0000000000000..c07cd21514e1b --- /dev/null +++ b/ci-operator/step-registry/ipi/azurestack/post/ipi-azurestack-post-chain.yaml @@ -0,0 +1,7 @@ +chain: + as: ipi-azurestack-post + steps: + - chain: ipi-deprovision + documentation: |- + The IPI cleanup step contains all steps that deprovision an OpenShift + cluster on Azure Stack Hub, provisioned by the `ipi-azurestack-post` chain. diff --git a/ci-operator/step-registry/ipi/azurestack/pre/OWNERS b/ci-operator/step-registry/ipi/azurestack/pre/OWNERS new file mode 100644 index 0000000000000..d3f5334072603 --- /dev/null +++ b/ci-operator/step-registry/ipi/azurestack/pre/OWNERS @@ -0,0 +1,8 @@ +approvers: +- staebler +- rna-afk +- patrickdillon +- mtnbikenc +- e-tienne +- jhixson74 +- jstuever diff --git a/ci-operator/step-registry/ipi/azurestack/pre/ipi-azurestack-pre-chain.metadata.json b/ci-operator/step-registry/ipi/azurestack/pre/ipi-azurestack-pre-chain.metadata.json new file mode 100644 index 0000000000000..4e29ae6ea617d --- /dev/null +++ b/ci-operator/step-registry/ipi/azurestack/pre/ipi-azurestack-pre-chain.metadata.json @@ -0,0 +1,14 @@ +{ + "path": "ipi/azurestack/pre/ipi-azurestack-pre-chain.yaml", + "owners": { + "approvers": [ + "staebler", + "rna-afk", + "patrickdillon", + "mtnbikenc", + "e-tienne", + "jhixson74", + "jstuever" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/ipi/azurestack/pre/ipi-azurestack-pre-chain.yaml b/ci-operator/step-registry/ipi/azurestack/pre/ipi-azurestack-pre-chain.yaml new file mode 100644 index 0000000000000..238bc4a9a6273 --- /dev/null +++ b/ci-operator/step-registry/ipi/azurestack/pre/ipi-azurestack-pre-chain.yaml @@ -0,0 +1,8 @@ +chain: + as: ipi-azurestack-pre + steps: + - chain: ipi-conf-azurestack + - chain: ipi-install + documentation: |- + The IPI setup step contains all steps that provision an OpenShift cluster + with a default configuration on Azure Stack Hub . diff --git a/ci-operator/step-registry/ipi/conf/azurestack/OWNERS b/ci-operator/step-registry/ipi/conf/azurestack/OWNERS new file mode 100644 index 0000000000000..d3f5334072603 --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/azurestack/OWNERS @@ -0,0 +1,8 @@ +approvers: +- staebler +- rna-afk +- patrickdillon +- mtnbikenc +- e-tienne +- jhixson74 +- jstuever diff --git a/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-chain.metadata.json b/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-chain.metadata.json new file mode 100644 index 0000000000000..4fbed0f97e613 --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-chain.metadata.json @@ -0,0 +1,14 @@ +{ + "path": "ipi/conf/azurestack/ipi-conf-azurestack-chain.yaml", + "owners": { + "approvers": [ + "staebler", + "rna-afk", + "patrickdillon", + "mtnbikenc", + "e-tienne", + "jhixson74", + "jstuever" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-chain.yaml b/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-chain.yaml new file mode 100644 index 0000000000000..dae52ee77e9bf --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-chain.yaml @@ -0,0 +1,8 @@ +chain: + as: ipi-conf-azurestack + steps: + - ref: ipi-conf + - ref: ipi-conf-azurestack + - ref: ipi-install-monitoringpvc + documentation: |- + The IPI configure step chain generates the install-config.yaml file based on the cluster profile and optional input files. diff --git a/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-commands.sh b/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-commands.sh new file mode 100644 index 0000000000000..f35658363f4f0 --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-commands.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +CONFIG="${SHARED_DIR}/install-config.yaml" +CLUSTER_NAME="${NAMESPACE}" +REGION="ppe3" +echo "Azure region: ${REGION}" + +workers=3 +if [[ "${SIZE_VARIANT}" == "compact" ]]; then + workers=0 +fi +master_type=null +if [[ "${SIZE_VARIANT}" == "xlarge" ]]; then + master_type=Standard_D32_v3 +elif [[ "${SIZE_VARIANT}" == "large" ]]; then + master_type=Standard_D16_v3 +elif [[ "${SIZE_VARIANT}" == "compact" ]]; then + master_type=Standard_D8_v3 +fi +echo $master_type + +ENDPOINT="${AZURESTACK_ENDPOINT}" +echo "ASH ARM Endpoint: ${ENDPOINT}" +cp "/var/run/azurestack-cluster-secrets/service-principal" "${SHARED_DIR}/osServicePrincipal.json" + +cat >> "${CONFIG}" << EOF +baseDomain: ppe.azurestack.devcluster.openshift.com +credentialsMode: Manual +metadata: + name: ${CLUSTER_NAME} +platform: + azure: + baseDomainResourceGroupName: openshiftInstallerRG + region: ${REGION} + cloudName: AzureStackCloud + armEndpoint: ${ENDPOINT} +controlPlane: + name: master +compute: +- name: worker + replicas: ${workers} +EOF + +echo "${AZURESTACK_ENDPOINT}" >> ${SHARED_DIR}/AZURESTACK_ENDPOINT +echo "${SUFFIX_ENDPOINT}" >> ${SHARED_DIR}/SUFFIX_ENDPOINT \ No newline at end of file diff --git a/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-ref.metadata.json b/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-ref.metadata.json new file mode 100644 index 0000000000000..e226fcee423e9 --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-ref.metadata.json @@ -0,0 +1,14 @@ +{ + "path": "ipi/conf/azurestack/ipi-conf-azurestack-ref.yaml", + "owners": { + "approvers": [ + "staebler", + "rna-afk", + "patrickdillon", + "mtnbikenc", + "e-tienne", + "jhixson74", + "jstuever" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-ref.yaml b/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-ref.yaml new file mode 100644 index 0000000000000..23f57a119d7b3 --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/azurestack/ipi-conf-azurestack-ref.yaml @@ -0,0 +1,36 @@ +ref: + as: ipi-conf-azurestack + from: upi-installer + commands: ipi-conf-azurestack-commands.sh + resources: + requests: + cpu: 10m + memory: 100Mi + credentials: + - namespace: test-credentials + name: azurestack-cluster-secrets + mount_path: /var/run/azurestack-cluster-secrets + env: + - name: SIZE_VARIANT + default: "" + documentation: |- + The size of the cluster in one of our supported t-shirt values that is standard across all CI environments. + + The sizes are: + * "" (default) - 4 vCPU, 16GB control plane nodes, default workers + * "compact" - 8 vCPU, 32GB control plane nodes, no workers + * "large" - 16 vCPU, 64GB+ control plane nodes, default workers, suitable for clusters up to 250 nodes + * "xlarge" - 32 vCPU, 128GB+ control plane nodes, default workers, suitable for clusters up to 1000 nodes + + These sizes are roughly consistent across all cloud providers, but we may not be able to instantiate some sizes + in some regions or accounts due to quota issues. + - name: AZURESTACK_ENDPOINT + default: 'https://management.ppe3.stackpoc.com' + documentation: |- + The Azure Stack Hub Endpoint to connect to. + - name: SUFFIX_ENDPOINT + default: "ppe3.stackpoc.com" + documentation: |- + The endpoint suffix for storage accounts. + documentation: |- + The IPI ASH configure step generates the Azure Stack Hub specific install-config.yaml contents based on the cluster profile and optional input files. diff --git a/ci-operator/step-registry/ipi/deprovision/deprovision/ipi-deprovision-deprovision-commands.sh b/ci-operator/step-registry/ipi/deprovision/deprovision/ipi-deprovision-deprovision-commands.sh index 8bff808676c05..b30f9045958fd 100644 --- a/ci-operator/step-registry/ipi/deprovision/deprovision/ipi-deprovision-deprovision-commands.sh +++ b/ci-operator/step-registry/ipi/deprovision/deprovision/ipi-deprovision-deprovision-commands.sh @@ -18,6 +18,12 @@ if [[ ! -s "${SHARED_DIR}/metadata.json" ]]; then exit fi +echo ${SHARED_DIR}/metadata.json + +if [[ "${CLUSTER_TYPE}" == "azurestack" ]]; then + export AZURE_AUTH_LOCATION=$SHARED_DIR/osServicePrincipal.json +fi + cp -ar "${SHARED_DIR}" /tmp/installer # TODO: remove once BZ#1926093 is done and backported diff --git a/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh b/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh index 53ab81c1a4d0c..bb30407764e7d 100755 --- a/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh +++ b/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh @@ -62,6 +62,7 @@ fi case "${CLUSTER_TYPE}" in aws|aws-arm64) export AWS_SHARED_CREDENTIALS_FILE=${CLUSTER_PROFILE_DIR}/.awscred;; azure4) export AZURE_AUTH_LOCATION=${CLUSTER_PROFILE_DIR}/osServicePrincipal.json;; +azurestack) export AZURE_AUTH_LOCATION=${SHARED_DIR}/osServicePrincipal.json;; gcp) export GOOGLE_CLOUD_KEYFILE_JSON=${CLUSTER_PROFILE_DIR}/gce.json;; kubevirt) export KUBEVIRT_KUBECONFIG=${HOME}/.kube/config;; vsphere) ;; diff --git a/ci-operator/step-registry/openshift/e2e/azurestack/OWNERS b/ci-operator/step-registry/openshift/e2e/azurestack/OWNERS new file mode 100644 index 0000000000000..d3f5334072603 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/azurestack/OWNERS @@ -0,0 +1,8 @@ +approvers: +- staebler +- rna-afk +- patrickdillon +- mtnbikenc +- e-tienne +- jhixson74 +- jstuever diff --git a/ci-operator/step-registry/openshift/e2e/azurestack/openshift-e2e-azurestack-workflow.metadata.json b/ci-operator/step-registry/openshift/e2e/azurestack/openshift-e2e-azurestack-workflow.metadata.json new file mode 100644 index 0000000000000..938ba9a81efb1 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/azurestack/openshift-e2e-azurestack-workflow.metadata.json @@ -0,0 +1,14 @@ +{ + "path": "openshift/e2e/azurestack/openshift-e2e-azurestack-workflow.yaml", + "owners": { + "approvers": [ + "staebler", + "rna-afk", + "patrickdillon", + "mtnbikenc", + "e-tienne", + "jhixson74", + "jstuever" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift/e2e/azurestack/openshift-e2e-azurestack-workflow.yaml b/ci-operator/step-registry/openshift/e2e/azurestack/openshift-e2e-azurestack-workflow.yaml new file mode 100644 index 0000000000000..e709eacdb799d --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/azurestack/openshift-e2e-azurestack-workflow.yaml @@ -0,0 +1,12 @@ +workflow: + as: openshift-e2e-azurestack + steps: + pre: + - chain: ipi-azurestack-pre + test: + - ref: openshift-e2e-test + post: + - chain: gather-core-dump + - chain: ipi-azurestack-post + documentation: |- + The Openshift E2E Azure workflow executes the common end-to-end test suite on Azure with a default cluster configuration. diff --git a/ci-operator/step-registry/openshift/e2e/azurestack/upi/OWNERS b/ci-operator/step-registry/openshift/e2e/azurestack/upi/OWNERS new file mode 100644 index 0000000000000..d3f5334072603 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/azurestack/upi/OWNERS @@ -0,0 +1,8 @@ +approvers: +- staebler +- rna-afk +- patrickdillon +- mtnbikenc +- e-tienne +- jhixson74 +- jstuever diff --git a/ci-operator/step-registry/openshift/e2e/azurestack/upi/openshift-e2e-azurestack-upi-workflow.metadata.json b/ci-operator/step-registry/openshift/e2e/azurestack/upi/openshift-e2e-azurestack-upi-workflow.metadata.json new file mode 100644 index 0000000000000..e9f7b444791b6 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/azurestack/upi/openshift-e2e-azurestack-upi-workflow.metadata.json @@ -0,0 +1,14 @@ +{ + "path": "openshift/e2e/azurestack/upi/openshift-e2e-azurestack-upi-workflow.yaml", + "owners": { + "approvers": [ + "staebler", + "rna-afk", + "patrickdillon", + "mtnbikenc", + "e-tienne", + "jhixson74", + "jstuever" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift/e2e/azurestack/upi/openshift-e2e-azurestack-upi-workflow.yaml b/ci-operator/step-registry/openshift/e2e/azurestack/upi/openshift-e2e-azurestack-upi-workflow.yaml new file mode 100644 index 0000000000000..05470e4394d47 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/azurestack/upi/openshift-e2e-azurestack-upi-workflow.yaml @@ -0,0 +1,12 @@ +workflow: + as: openshift-e2e-azurestack-upi + steps: + pre: + - chain: upi-azurestack-pre + test: + - ref: openshift-e2e-test + post: + - chain: gather-core-dump + - chain: upi-azurestack-post + documentation: |- + The Openshift E2E Azure workflow executes the common end-to-end test suite on Azure with a default cluster configuration. diff --git a/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-commands.sh b/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-commands.sh index 024e6bd5aa3c1..161ad6e913d66 100755 --- a/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-commands.sh +++ b/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-commands.sh @@ -121,6 +121,10 @@ aws|aws-arm64) export KUBE_SSH_USER=core ;; azure4) export TEST_PROVIDER=azure;; +azurestack) + export TEST_PROVIDER=azure + export AZURE_AUTH_LOCATION=${SHARED_DIR}/osServicePrincipal.json + ;; vsphere) export TEST_PROVIDER=vsphere;; openstack*) # shellcheck disable=SC1090 diff --git a/ci-operator/step-registry/upi/azurestack/OWNERS b/ci-operator/step-registry/upi/azurestack/OWNERS new file mode 100644 index 0000000000000..d3f5334072603 --- /dev/null +++ b/ci-operator/step-registry/upi/azurestack/OWNERS @@ -0,0 +1,8 @@ +approvers: +- staebler +- rna-afk +- patrickdillon +- mtnbikenc +- e-tienne +- jhixson74 +- jstuever diff --git a/ci-operator/step-registry/upi/azurestack/post/OWNERS b/ci-operator/step-registry/upi/azurestack/post/OWNERS new file mode 100644 index 0000000000000..d3f5334072603 --- /dev/null +++ b/ci-operator/step-registry/upi/azurestack/post/OWNERS @@ -0,0 +1,8 @@ +approvers: +- staebler +- rna-afk +- patrickdillon +- mtnbikenc +- e-tienne +- jhixson74 +- jstuever diff --git a/ci-operator/step-registry/upi/azurestack/post/upi-azurestack-post-chain.metadata.json b/ci-operator/step-registry/upi/azurestack/post/upi-azurestack-post-chain.metadata.json new file mode 100644 index 0000000000000..25cfb832dafbb --- /dev/null +++ b/ci-operator/step-registry/upi/azurestack/post/upi-azurestack-post-chain.metadata.json @@ -0,0 +1,14 @@ +{ + "path": "upi/azurestack/post/upi-azurestack-post-chain.yaml", + "owners": { + "approvers": [ + "staebler", + "rna-afk", + "patrickdillon", + "mtnbikenc", + "e-tienne", + "jhixson74", + "jstuever" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/upi/azurestack/post/upi-azurestack-post-chain.yaml b/ci-operator/step-registry/upi/azurestack/post/upi-azurestack-post-chain.yaml new file mode 100644 index 0000000000000..e503077dabf9f --- /dev/null +++ b/ci-operator/step-registry/upi/azurestack/post/upi-azurestack-post-chain.yaml @@ -0,0 +1,7 @@ +chain: + as: upi-azurestack-post + steps: + - chain: ipi-deprovision + - ref: upi-deprovision-azurestack + documentation: >- + This chain deprovisions all the components created by the upi-azurestack-pre chain. diff --git a/ci-operator/step-registry/upi/azurestack/pre/OWNERS b/ci-operator/step-registry/upi/azurestack/pre/OWNERS new file mode 100644 index 0000000000000..d3f5334072603 --- /dev/null +++ b/ci-operator/step-registry/upi/azurestack/pre/OWNERS @@ -0,0 +1,8 @@ +approvers: +- staebler +- rna-afk +- patrickdillon +- mtnbikenc +- e-tienne +- jhixson74 +- jstuever diff --git a/ci-operator/step-registry/upi/azurestack/pre/upi-azurestack-pre-chain.metadata.json b/ci-operator/step-registry/upi/azurestack/pre/upi-azurestack-pre-chain.metadata.json new file mode 100644 index 0000000000000..ba4490671ff7c --- /dev/null +++ b/ci-operator/step-registry/upi/azurestack/pre/upi-azurestack-pre-chain.metadata.json @@ -0,0 +1,14 @@ +{ + "path": "upi/azurestack/pre/upi-azurestack-pre-chain.yaml", + "owners": { + "approvers": [ + "staebler", + "rna-afk", + "patrickdillon", + "mtnbikenc", + "e-tienne", + "jhixson74", + "jstuever" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/upi/azurestack/pre/upi-azurestack-pre-chain.yaml b/ci-operator/step-registry/upi/azurestack/pre/upi-azurestack-pre-chain.yaml new file mode 100644 index 0000000000000..dde5eca5d6779 --- /dev/null +++ b/ci-operator/step-registry/upi/azurestack/pre/upi-azurestack-pre-chain.yaml @@ -0,0 +1,9 @@ +chain: + as: upi-azurestack-pre + steps: + - ref: ipi-install-rbac + - chain: ipi-conf-azurestack + - ref: upi-install-azurestack + - ref: ipi-install-times-collection + documentation: >- + This chain contains all of the steps to provision an OpenShift cluster using the Azure Stack Hub UPI workflow. diff --git a/ci-operator/step-registry/upi/azurestack/upi-azurestack-workflow.metadata.json b/ci-operator/step-registry/upi/azurestack/upi-azurestack-workflow.metadata.json new file mode 100644 index 0000000000000..3133a8613829c --- /dev/null +++ b/ci-operator/step-registry/upi/azurestack/upi-azurestack-workflow.metadata.json @@ -0,0 +1,14 @@ +{ + "path": "upi/azurestack/upi-azurestack-workflow.yaml", + "owners": { + "approvers": [ + "staebler", + "rna-afk", + "patrickdillon", + "mtnbikenc", + "e-tienne", + "jhixson74", + "jstuever" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/upi/azurestack/upi-azurestack-workflow.yaml b/ci-operator/step-registry/upi/azurestack/upi-azurestack-workflow.yaml new file mode 100644 index 0000000000000..958e27a5c95b6 --- /dev/null +++ b/ci-operator/step-registry/upi/azurestack/upi-azurestack-workflow.yaml @@ -0,0 +1,15 @@ +workflow: + as: upi-azurestack + steps: + pre: + - chain: upi-azurestack-pre + post: + - chain: upi-azurestack-post + documentation: |- + The UPI workflow provides pre- and post- steps that provision and + deprovision an OpenShift cluster with a default configuration on Azure + Stack hub allowing job authors to inject their own end-to-end test logic. + + All modifications to this workflow should be done by modifying the + `upi-azurestack-{pre,post}` chains to allow other workflows to mimic and extend + this base workflow without a need to backport changes. diff --git a/ci-operator/step-registry/upi/deprovision/azurestack/OWNERS b/ci-operator/step-registry/upi/deprovision/azurestack/OWNERS new file mode 100644 index 0000000000000..d3f5334072603 --- /dev/null +++ b/ci-operator/step-registry/upi/deprovision/azurestack/OWNERS @@ -0,0 +1,8 @@ +approvers: +- staebler +- rna-afk +- patrickdillon +- mtnbikenc +- e-tienne +- jhixson74 +- jstuever diff --git a/ci-operator/step-registry/upi/deprovision/azurestack/upi-deprovision-azurestack-commands.sh b/ci-operator/step-registry/upi/deprovision/azurestack/upi-deprovision-azurestack-commands.sh new file mode 100644 index 0000000000000..ea5aeeb328cda --- /dev/null +++ b/ci-operator/step-registry/upi/deprovision/azurestack/upi-deprovision-azurestack-commands.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM + +TENANT_ID=$(cat ${SHARED_DIR}/TENANT_ID) +APP_ID=$(cat ${SHARED_DIR}/APP_ID) +AAD_CLIENT_SECRET=$(cat ${SHARED_DIR}/AAD_CLIENT_SECRET) + +AZURESTACK_ENDPOINT=$(cat ${SHARED_DIR}/AZURESTACK_ENDPOINT) +SUFFIX_ENDPOINT=$(cat ${SHARED_DIR}/SUFFIX_ENDPOINT) +RESOURCE_GROUP=$(cat "${SHARED_DIR}/RESOURCE_GROUP_NAME") + +az cloud register \ + -n PPE \ + --endpoint-resource-manager "${AZURESTACK_ENDPOINT}" \ + --suffix-storage-endpoint "${SUFFIX_ENDPOINT}" +az cloud set -n PPE +az cloud update --profile 2019-03-01-hybrid +az login --service-principal -u $APP_ID -p $AAD_CLIENT_SECRET --tenant $TENANT_ID > /dev/null + +az group delete --help +az group delete --resource-group $RESOURCE_GROUP -y +echo "Deleted successfully!" \ No newline at end of file diff --git a/ci-operator/step-registry/upi/deprovision/azurestack/upi-deprovision-azurestack-ref.metadata.json b/ci-operator/step-registry/upi/deprovision/azurestack/upi-deprovision-azurestack-ref.metadata.json new file mode 100644 index 0000000000000..1a5f6586d343b --- /dev/null +++ b/ci-operator/step-registry/upi/deprovision/azurestack/upi-deprovision-azurestack-ref.metadata.json @@ -0,0 +1,14 @@ +{ + "path": "upi/deprovision/azurestack/upi-deprovision-azurestack-ref.yaml", + "owners": { + "approvers": [ + "staebler", + "rna-afk", + "patrickdillon", + "mtnbikenc", + "e-tienne", + "jhixson74", + "jstuever" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/upi/deprovision/azurestack/upi-deprovision-azurestack-ref.yaml b/ci-operator/step-registry/upi/deprovision/azurestack/upi-deprovision-azurestack-ref.yaml new file mode 100644 index 0000000000000..63334838ce305 --- /dev/null +++ b/ci-operator/step-registry/upi/deprovision/azurestack/upi-deprovision-azurestack-ref.yaml @@ -0,0 +1,12 @@ +ref: + as: upi-deprovision-azurestack + from: upi-installer + grace_period: 10m + commands: upi-deprovision-azurestack-commands.sh + resources: + requests: + cpu: 10m + memory: 100Mi + documentation: >- + This step deprovisions the gcloud deployments created by upi-install-azurestack. + It requires the ipi-deprovision step already be executed against the cluster. diff --git a/ci-operator/step-registry/upi/install/azurestack/OWNERS b/ci-operator/step-registry/upi/install/azurestack/OWNERS new file mode 100644 index 0000000000000..d3f5334072603 --- /dev/null +++ b/ci-operator/step-registry/upi/install/azurestack/OWNERS @@ -0,0 +1,8 @@ +approvers: +- staebler +- rna-afk +- patrickdillon +- mtnbikenc +- e-tienne +- jhixson74 +- jstuever diff --git a/ci-operator/step-registry/upi/install/azurestack/upi-install-azurestack-commands.sh b/ci-operator/step-registry/upi/install/azurestack/upi-install-azurestack-commands.sh new file mode 100644 index 0000000000000..fba545929c8fb --- /dev/null +++ b/ci-operator/step-registry/upi/install/azurestack/upi-install-azurestack-commands.sh @@ -0,0 +1,297 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM +export HOME=/tmp + +if [[ -z "$RELEASE_IMAGE_LATEST" ]]; then + echo "RELEASE_IMAGE_LATEST is an empty string, exiting" + exit 1 +fi + +# Ensure ignition assets are configured with the correct invoker to track CI jobs. +export OPENSHIFT_INSTALL_INVOKER="openshift-internal-ci/${JOB_NAME_SAFE}/${BUILD_ID}" +export TEST_PROVIDER='azurestack' +export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${RELEASE_IMAGE_LATEST}" + +dir=/tmp/installer +mkdir "${dir}" +pushd "${dir}" +cp -t "${dir}" \ + "${SHARED_DIR}/install-config.yaml" + +if ! pip -V; then + echo "pip is not installed: installing" + if python -c "import sys; assert(sys.version_info >= (3,0))"; then + python -m ensurepip --user || easy_install --user 'pip' + else + echo "python < 3, installing pip<21" + python -m ensurepip --user || easy_install --user 'pip<21' + fi +fi + +echo "Installing python modules: yaml" +python3 -c "import yaml" || pip3 install --user pyyaml +sed -i "s|ppe.azurestack.devcluster.openshift.com|ppe.upi.azurestack.devcluster.openshift.com|g" install-config.yaml + +CLUSTER_NAME=$(python3 -c 'import yaml;data = yaml.full_load(open("install-config.yaml"));print(data["metadata"]["name"])') +AAD_CLIENT_ID=$(jq -r .clientId ${SHARED_DIR}/osServicePrincipal.json) +AZURE_REGION=ppe3 +SSH_KEY=$(python3 -c 'import yaml;data = yaml.full_load(open("install-config.yaml"));print(data["sshKey"])') +BASE_DOMAIN=$(python3 -c 'import yaml;data = yaml.full_load(open("install-config.yaml"));print(data["baseDomain"])') +TENANT_ID=$(jq -r .tenantId ${SHARED_DIR}/osServicePrincipal.json) +AAD_CLIENT_SECRET=$(jq -r .clientSecret ${SHARED_DIR}/osServicePrincipal.json) +APP_ID=$(jq -r .clientId ${SHARED_DIR}/osServicePrincipal.json) + +export CLUSTER_NAME +export AAD_CLIENT_ID +export AZURE_REGION +export SSH_KEY +export TENANT_ID +export BASE_DOMAIN +export AAD_CLIENT_SECRET +export APP_ID + +echo $TENANT_ID >> ${SHARED_DIR}/TENANT_ID +echo $AAD_CLIENT_SECRET >> ${SHARED_DIR}/AAD_CLIENT_SECRET +echo $APP_ID >> ${SHARED_DIR}/APP_ID + +AZURESTACK_ENDPOINT=$(cat ${SHARED_DIR}/AZURESTACK_ENDPOINT) +SUFFIX_ENDPOINT=$(cat ${SHARED_DIR}/SUFFIX_ENDPOINT) + +az cloud register \ + -n PPE \ + --endpoint-resource-manager "${AZURESTACK_ENDPOINT}" \ + --suffix-storage-endpoint "${SUFFIX_ENDPOINT}" +az cloud set -n PPE +az cloud update --profile 2019-03-01-hybrid +az login --service-principal -u $APP_ID -p $AAD_CLIENT_SECRET --tenant $TENANT_ID > /dev/null + +# shellcheck disable=SC2034 +SUBSCRIPTION_ID=$(az account show | jq -r .id) +export SUBSCRIPTION_ID +export AZURE_AUTH_LOCATION="${SHARED_DIR}/osServicePrincipal.json" + +# remove workers from the install config so the mco won't try to create them +python3 -c ' +import yaml; +path = "install-config.yaml"; +data = yaml.full_load(open(path)); +data["compute"][0]["replicas"] = 0; +open(path, "w").write(yaml.dump(data, default_flow_style=False))' + +openshift-install create manifests + +# we don't want to create any machine* objects +rm -f openshift/99_openshift-cluster-api_master-machines-*.yaml +rm -f openshift/99_openshift-cluster-api_worker-machineset-*.yaml + +RESOURCE_GROUP=$(python3 -c 'import yaml;data = yaml.full_load(open("manifests/cluster-infrastructure-02-config.yml"));print(data["status"]["platformStatus"]["azure"]["resourceGroupName"])') +oc adm release extract $RELEASE_IMAGE_LATEST --credentials-requests --cloud=azure --to=credentials-request +ls credentials-request +files=$(ls credentials-request) +for f in $files +do + SECRET_NAME=$(python3 -c 'import yaml;data = yaml.full_load(open("credentials-request/'${f}'"));print(data["spec"]["secretRef"]["name"])') + SECRET_NAMESPACE=$(python3 -c 'import yaml;data = yaml.full_load(open("credentials-request/'${f}'"));print(data["spec"]["secretRef"]["namespace"])') + filename=${f/request/secret} + cat >> "manifests/$filename" << EOF +apiVersion: v1 +kind: Secret +metadata: + name: ${SECRET_NAME} + namespace: ${SECRET_NAMESPACE} +stringData: + azure_subscription_id: ${SUBSCRIPTION_ID} + azure_client_id: ${AAD_CLIENT_ID} + azure_client_secret: ${AAD_CLIENT_SECRET} + azure_tenant_id: ${TENANT_ID} + azure_resource_prefix: ${CLUSTER_NAME} + azure_resourcegroup: ${RESOURCE_GROUP} + azure_region: ${AZURE_REGION} +EOF +done + +cat >> manifests/cco-configmap.yaml < "${SHARED_DIR}/RESOURCE_GROUP_NAME" + +openshift-install create ignition-configs & + +set +e +wait "$!" +ret="$?" +set -e + +cp "${dir}/.openshift_install.log" "${ARTIFACT_DIR}/.openshift_install.log" + +if [ $ret -ne 0 ]; then + exit "$ret" +fi + +tar -czf "${SHARED_DIR}/.openshift_install_state.json.tgz" ".openshift_install_state.json" + +export SSH_PRIV_KEY_PATH="${CLUSTER_PROFILE_DIR}/ssh-privatekey" +export OPENSHIFT_INSTALL_INVOKER="openshift-internal-ci/${JOB_NAME_SAFE}/${BUILD_ID}" + +date "+%F %X" > "${SHARED_DIR}/CLUSTER_INSTALL_START_TIME" +RESOURCE_GROUP=$(cat "${SHARED_DIR}/RESOURCE_GROUP_NAME") + +az group create --name "$RESOURCE_GROUP" --location "$AZURE_REGION" + +KUBECONFIG="${dir}/auth/kubeconfig" +export KUBECONFIG + +az storage account create -g "$RESOURCE_GROUP" --location "$AZURE_REGION" --name "${INFRA_ID}sa" --kind Storage --sku Standard_LRS +ACCOUNT_KEY=$(az storage account keys list -g "$RESOURCE_GROUP" --account-name "${INFRA_ID}sa" --query "[0].value" -o tsv) + +az storage container create --name files --account-name "${INFRA_ID}sa" --public-access blob --account-key "$ACCOUNT_KEY" +az storage blob upload --account-name "${INFRA_ID}sa" --account-key "$ACCOUNT_KEY" -c "files" -f "bootstrap.ign" -n "bootstrap.ign" + +AZURESTACK_UPI_LOCATION="/var/lib/openshift-install/upi/azurestack" +az deployment group create -g "$RESOURCE_GROUP" \ + --template-file "${AZURESTACK_UPI_LOCATION}/01_vnet.json" \ + --parameters baseName="$INFRA_ID" + +VHD_BLOB_URL="https://rhcossa.blob.ppe3.stackpoc.com/vhd/rhcos-49-84-202108221651.vhd" +az deployment group create -g "$RESOURCE_GROUP" \ + --template-file "${AZURESTACK_UPI_LOCATION}/02_storage.json" \ + --parameters vhdBlobURL="$VHD_BLOB_URL" \ + --parameters baseName="$INFRA_ID" + +az deployment group create -g "$RESOURCE_GROUP" \ + --template-file "${AZURESTACK_UPI_LOCATION}/03_infra.json" \ + --parameters baseName="$INFRA_ID" + +az network dns zone create -g "$RESOURCE_GROUP" -n "${CLUSTER_NAME}.${BASE_DOMAIN}" +PUBLIC_IP=$(az network public-ip list -g "$RESOURCE_GROUP" --query "[?name=='${INFRA_ID}-master-pip'] | [0].ipAddress" -o tsv) +az network dns record-set a add-record -g "$RESOURCE_GROUP" -z "${CLUSTER_NAME}.${BASE_DOMAIN}" -n api -a "$PUBLIC_IP" --ttl 60 +az network dns record-set a add-record -g "$RESOURCE_GROUP" -z "${CLUSTER_NAME}.${BASE_DOMAIN}" -n api-int -a "$PUBLIC_IP" --ttl 60 + +BOOTSTRAP_URL=$(az storage blob url --account-name "${INFRA_ID}sa" --account-key "$ACCOUNT_KEY" -c "files" -n "bootstrap.ign" -o tsv) +BOOTSTRAP_IGNITION=$(jq -rcnM --arg v "3.2.0" --arg url "$BOOTSTRAP_URL" '{ignition:{version:$v,config:{replace:{source:$url}}}}' | base64 | tr -d '\n') + +az deployment group create --verbose -g "$RESOURCE_GROUP" \ + --template-file "${AZURESTACK_UPI_LOCATION}/04_bootstrap.json" \ + --parameters bootstrapIgnition="$BOOTSTRAP_IGNITION" \ + --parameters sshKeyData="$SSH_KEY" \ + --parameters baseName="$INFRA_ID" \ + --parameters diagnosticsStorageAccountName="${INFRA_ID}sa" + +MASTER_IGNITION=$(cat master.ign | base64 | tr -d '\n') +az deployment group create -g "$RESOURCE_GROUP" \ + --template-file "${AZURESTACK_UPI_LOCATION}/05_masters.json" \ + --parameters masterIgnition="$MASTER_IGNITION" \ + --parameters sshKeyData="$SSH_KEY" \ + --parameters baseName="$INFRA_ID" \ + --parameters diagnosticsStorageAccountName="${INFRA_ID}sa" + +echo "$(date -u --rfc-3339=seconds) - Monitoring for bootstrap to complete" +openshift-install wait-for bootstrap-complete & + +set +e +wait "$!" +ret="$?" +set -e + +if [ "$ret" -ne 0 ]; then + set +e + # Attempt to gather bootstrap logs. + echo "$(date -u --rfc-3339=seconds) - Bootstrap failed, attempting to gather bootstrap logs..." + openshift-install "--dir=${dir}" gather bootstrap --key "${SSH_PRIV_KEY_PATH}" "${GATHER_BOOTSTRAP_ARGS[@]}" + sed 's/password: .*/password: REDACTED/' "${dir}/.openshift_install.log" >>"${ARTIFACT_DIR}/.openshift_install.log" + cp log-bundle-*.tar.gz "${ARTIFACT_DIR}" + set -e + exit "$ret" +fi + +az network nsg rule delete -g "$RESOURCE_GROUP" --nsg-name "${INFRA_ID}"-nsg --name bootstrap_ssh_in +az vm stop -g "$RESOURCE_GROUP" --name "${INFRA_ID}"-bootstrap +az vm deallocate -g "$RESOURCE_GROUP" --name "${INFRA_ID}"-bootstrap +az vm delete -g "$RESOURCE_GROUP" --name "${INFRA_ID}"-bootstrap --yes +az disk delete -g "$RESOURCE_GROUP" --name "${INFRA_ID}"-bootstrap_OSDisk --no-wait --yes +az network nic delete -g "$RESOURCE_GROUP" --name "${INFRA_ID}"-bootstrap-nic --no-wait +az storage blob delete --account-key "$ACCOUNT_KEY" --account-name "${INFRA_ID}sa" --container-name files --name bootstrap.ign +az network public-ip delete -g "$RESOURCE_GROUP" --name "${INFRA_ID}"-bootstrap-ssh-pip + +WORKER_IGNITION=$(cat worker.ign | base64 | tr -d '\n') +az deployment group create -g "$RESOURCE_GROUP" \ + --template-file "${AZURESTACK_UPI_LOCATION}/06_workers.json" \ + --parameters workerIgnition="$WORKER_IGNITION" \ + --parameters sshKeyData="$SSH_KEY" \ + --parameters baseName="$INFRA_ID" \ + --parameters diagnosticsStorageAccountName="${INFRA_ID}sa" + +echo "$(date -u --rfc-3339=seconds) - Approving the CSR requests for nodes..." +function approve_csrs() { + while [[ ! -f /tmp/install-complete ]]; do + # even if oc get csr fails continue + oc get csr -ojson | jq -r '.items[] | select(.status == {} ) | .metadata.name' | xargs --no-run-if-empty oc adm certificate approve || true + sleep 15 & wait + done +} +approve_csrs & + +## Wait for the default-router to have an external ip...(and not ) + echo "$(date -u --rfc-3339=seconds) - Waiting for the default-router to have an external ip..." + set +e + ROUTER_IP="$(oc -n openshift-ingress get service router-default --no-headers | awk '{print $4}')" + while [[ "$ROUTER_IP" == "" || "$ROUTER_IP" == "" ]]; do + sleep 10; + ROUTER_IP="$(oc -n openshift-ingress get service router-default --no-headers | awk '{print $4}')" + done + set -e + +az network dns record-set a add-record -g $RESOURCE_GROUP -z ${CLUSTER_NAME}.${BASE_DOMAIN} -n *.apps -a $ROUTER_IP --ttl 300 + +echo "$(date -u --rfc-3339=seconds) - Monitoring for cluster completion..." +openshift-install --dir="${dir}" wait-for install-complete 2>&1 | grep --line-buffered -v 'password\|X-Auth-Token\|UserData:' & + +set +e +wait "$!" +ret="$?" +set -e + +echo "$(date +%s)" > "${SHARED_DIR}/TEST_TIME_INSTALL_END" +date "+%F %X" > "${SHARED_DIR}/CLUSTER_INSTALL_END_TIME" + +sed 's/password: .*/password: REDACTED/' "${dir}/.openshift_install.log" >>"${ARTIFACT_DIR}/.openshift_install.log" + +if [ $ret -ne 0 ]; then + exit "$ret" +fi + +cp -t "${SHARED_DIR}" \ + "${dir}/auth/kubeconfig" +popd +touch /tmp/install-complete diff --git a/ci-operator/step-registry/upi/install/azurestack/upi-install-azurestack-ref.metadata.json b/ci-operator/step-registry/upi/install/azurestack/upi-install-azurestack-ref.metadata.json new file mode 100644 index 0000000000000..31829ae4ffbed --- /dev/null +++ b/ci-operator/step-registry/upi/install/azurestack/upi-install-azurestack-ref.metadata.json @@ -0,0 +1,14 @@ +{ + "path": "upi/install/azurestack/upi-install-azurestack-ref.yaml", + "owners": { + "approvers": [ + "staebler", + "rna-afk", + "patrickdillon", + "mtnbikenc", + "e-tienne", + "jhixson74", + "jstuever" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/upi/install/azurestack/upi-install-azurestack-ref.yaml b/ci-operator/step-registry/upi/install/azurestack/upi-install-azurestack-ref.yaml new file mode 100644 index 0000000000000..68624bdf48444 --- /dev/null +++ b/ci-operator/step-registry/upi/install/azurestack/upi-install-azurestack-ref.yaml @@ -0,0 +1,11 @@ +ref: + as: upi-install-azurestack + from: upi-installer + grace_period: 10m + commands: upi-install-azurestack-commands.sh + resources: + requests: + cpu: 10m + memory: 100Mi + documentation: >- + This step deploys a UPI cluster to the CI Azure Stack Hub project. diff --git a/clusters/app.ci/prow/03_deployment/boskos_reaper.yaml b/clusters/app.ci/prow/03_deployment/boskos_reaper.yaml index cb8792601cb50..b5ac0602b4c9f 100644 --- a/clusters/app.ci/prow/03_deployment/boskos_reaper.yaml +++ b/clusters/app.ci/prow/03_deployment/boskos_reaper.yaml @@ -23,5 +23,5 @@ spec: image: gcr.io/k8s-staging-boskos/reaper:v20210823-ae371c6 args: - --boskos-url=http://boskos - - --resource-type=aws-quota-slice,azure4-quota-slice,azure-2-quota-slice,gcp-quota-slice,libvirt-ppc64le-quota-slice,libvirt-s390x-quota-slice,openstack-quota-slice,openstack-kuryr-quota-slice,openstack-vexxhost-quota-slice,openstack-vh-mecha-central-quota-slice,openstack-vh-mecha-az0-quota-slice,openstack-ppc64le-quota-slice,vsphere-quota-slice,ovirt-quota-slice,packet-quota-slice,packet-edge-quota-slice,kubevirt-quota-slice,openstack-osuosl-quota-slice,aws-cpaas-quota-slice,hypershift-quota-slice,alibaba-quota-slice,aws-2-quota-slice,aws-arm64-quota-slice,azurestack-quota-slice,gcp-openshift-gce-devel-ci-2-quota-slice,ibmcloud-quota-slice,metal-quota-slice,osd-ephemeral-quota-slice,ovirt-upgrade-quota-slice,aws-china-quota-slice,aws-usgov-quota-slice,vsphere-discon-quota-slice + - --resource-type=aws-quota-slice,azure4-quota-slice,azure-2-quota-slice,azurestack-quota-slice,gcp-quota-slice,libvirt-ppc64le-quota-slice,libvirt-s390x-quota-slice,openstack-quota-slice,openstack-kuryr-quota-slice,openstack-vexxhost-quota-slice,openstack-vh-mecha-central-quota-slice,openstack-vh-mecha-az0-quota-slice,openstack-ppc64le-quota-slice,vsphere-quota-slice,ovirt-quota-slice,packet-quota-slice,packet-edge-quota-slice,kubevirt-quota-slice,openstack-osuosl-quota-slice,aws-cpaas-quota-slice,hypershift-quota-slice,alibaba-quota-slice,aws-2-quota-slice,aws-arm64-quota-slice,azurestack-quota-slice,gcp-openshift-gce-devel-ci-2-quota-slice,ibmcloud-quota-slice,metal-quota-slice,osd-ephemeral-quota-slice,ovirt-upgrade-quota-slice,aws-china-quota-slice,aws-usgov-quota-slice,vsphere-discon-quota-slice - --target-state=free