diff --git a/Makefile b/Makefile index f276d4835fce7..718db9954e190 100644 --- a/Makefile +++ b/Makefile @@ -315,6 +315,15 @@ azure-secrets: oc create secret generic codecov-token --from-literal=upload=${CODECOV_UPLOAD_TOKEN} -o yaml --dry-run | oc apply -n azure -f - .PHONY: azure-secrets +azure4-secrets: + oc create secret generic cluster-secrets-azure4 \ + --from-file=cluster/test-deploy/azure4/osServicePrincipal.json \ + --from-file=cluster/test-deploy/azure4/pull-secret \ + --from-file=cluster/test-deploy/azure4/ssh-privatekey \ + --from-file=cluster/test-deploy/azure4/ssh-publickey \ + -o yaml --dry-run | oc apply -n ocp -f - +.PHONY: azure4-secrets + metering: $(MAKE) -C projects/metering .PHONY: metering diff --git a/ci-operator/SECRETS.md b/ci-operator/SECRETS.md index 5ff33615f537f..3c3d1a94311dc 100644 --- a/ci-operator/SECRETS.md +++ b/ci-operator/SECRETS.md @@ -54,6 +54,15 @@ currently exist: | `metrics-int.key` | Azure Geneva metrics authentication key | | `system-docker-config.json` | Root/node/system level docker config.json file, currently holding access registry.redhat.io | +#### `cluster-secrets-azure4` + +| Key | Description | +| ----------------------------------| ----------- | +| `osServicePrincipal.json` | Credentials for the Azure API. This is a json file that contains fields described in [upstream credentials doc](https://docs.microsoft.com/en-us/azure-stack/operator/azure-stack-create-service-principals#create-a-service-principal-using-a-client-secret). | +| `pull-secret` | Credentials for pulling OpenShift images from Quay and for authenticating to telemetry. Retrieved from [try.openshift.com](https://try.openshift.com) under the ccoleman+openshift-ci-test@redhat.com account, and has the service account token from the `ocp` namespace added with `oc registry login --to=/tmp/pull-secret -z default -n ocp`.| +| `ssh-privatekey` | Private half of the SSH key, for connecting to Azure VMs. | +| `ssh-publickey` | Public half of the SSH key, for connecting to Azure VMs. | + #### `cluster-secrets-vsphere` | Key | Description | diff --git a/ci-operator/config/openshift/installer/openshift-installer-master.yaml b/ci-operator/config/openshift/installer/openshift-installer-master.yaml index 2d038ada68c86..9a545876c9e91 100644 --- a/ci-operator/config/openshift/installer/openshift-installer-master.yaml +++ b/ci-operator/config/openshift/installer/openshift-installer-master.yaml @@ -128,6 +128,11 @@ tests: openshift_installer: cluster_profile: aws upgrade: false +- as: e2e-azure + commands: TEST_SUITE=openshift/conformance/parallel run-tests + openshift_installer: + cluster_profile: azure4 + upgrade: false - as: e2e-aws-upgrade commands: TEST_SUITE=all run-upgrade-tests openshift_installer: diff --git a/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml b/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml index 0f1d6e5d92d0a..e397c0955d353 100644 --- a/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml +++ b/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml @@ -350,6 +350,71 @@ presubmits: secret: secretName: sentry-dsn trigger: (?m)^/test( | .* )e2e-aws-upi,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - master + context: ci/prow/e2e-azure + decorate: true + decoration_config: + skip_cloning: true + labels: + ci-operator.openshift.io/prowgen-controlled: "true" + name: pull-ci-openshift-installer-master-e2e-azure + optional: true + rerun_command: /test e2e-azure + spec: + containers: + - args: + - --artifact-dir=$(ARTIFACTS) + - --give-pr-author-access-to-namespace=true + - --secret-dir=/usr/local/e2e-azure-cluster-profile + - --sentry-dsn-path=/etc/sentry-dsn/ci-operator + - --target=e2e-azure + - --template=/usr/local/e2e-azure + command: + - ci-operator + env: + - name: CLUSTER_TYPE + value: azure4 + - name: CONFIG_SPEC + valueFrom: + configMapKeyRef: + key: openshift-installer-master.yaml + name: ci-operator-master-configs + - name: JOB_NAME_SAFE + value: e2e-azure + - name: TEST_COMMAND + value: TEST_SUITE=openshift/conformance/parallel run-tests + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /usr/local/e2e-azure-cluster-profile + name: cluster-profile + - mountPath: /usr/local/e2e-azure + name: job-definition + subPath: cluster-launch-installer-e2e.yaml + - mountPath: /etc/sentry-dsn + name: sentry-dsn + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: cluster-profile + projected: + sources: + - secret: + name: cluster-secrets-azure4 + - configMap: + name: prow-job-cluster-launch-installer-e2e + name: job-definition + - name: sentry-dsn + secret: + secretName: sentry-dsn + trigger: (?m)^/test( | .* )e2e-azure,?($|\s.*) - agent: kubernetes always_run: false branches: diff --git a/ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml b/ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml index ba8e94c282ed4..1107fb9ba1571 100644 --- a/ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml +++ b/ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml @@ -19,8 +19,6 @@ parameters: - name: RELEASE_IMAGE_LATEST required: true - name: BASE_DOMAIN - value: origin-ci-int-aws.dev.rhcloud.com - required: true objects: @@ -97,6 +95,8 @@ objects: env: - name: AWS_SHARED_CREDENTIALS_FILE value: /tmp/cluster/.awscred + - name: AZURE_AUTH_LOCATION + value: /tmp/cluster/osServicePrincipal.json - name: ARTIFACT_DIR value: /tmp/artifacts - name: HOME @@ -154,6 +154,12 @@ objects: # TODO: make openshift-tests auto-discover this from cluster config export TEST_PROVIDER='{"type":"aws","region":"us-east-1","zone":"us-east-1a","multizone":true,"multimaster":true}' export KUBE_SSH_USER=core + elif [[ "${CLUSTER_TYPE}" == "azure" ]]; then + mkdir -p ~/.ssh + export PROVIDER_ARGS="-provider=azure -gce-zone=centralus" + # TODO: make openshift-tests auto-discover this from cluster config + export TEST_PROVIDER='{"type":"azure","region":"centralus","multizone":true,"multimaster":true}' + export KUBE_SSH_USER=core fi mkdir -p /tmp/output @@ -527,6 +533,10 @@ objects: value: /etc/openshift-installer/.awscred - name: AWS_REGION value: us-east-1 + - name: AZURE_AUTH_LOCATION + value: /etc/openshift-installer/osServicePrincipal.json + - name: AZURE_REGION + value: centralus - name: CLUSTER_NAME value: ${NAMESPACE}-${JOB_NAME_HASH} - name: BASE_DOMAIN @@ -567,8 +577,8 @@ objects: if [[ "${CLUSTER_TYPE}" == "aws" ]]; then cat > /tmp/artifacts/installer/install-config.yaml << EOF - apiVersion: v1beta4 - baseDomain: ${BASE_DOMAIN} + apiVersion: v1 + baseDomain: ${BASE_DOMAIN:-origin-ci-int-aws.dev.rhcloud.com} metadata: name: ${CLUSTER_NAME} controlPlane: @@ -597,6 +607,27 @@ objects: sshKey: | ${SSH_PUB_KEY} EOF + elif [[ "${CLUSTER_TYPE}" == "azure4" ]]; then + cat > /tmp/artifacts/installer/install-config.yaml << EOF + apiVersion: v1 + baseDomain: ${BASE_DOMAIN:-ci.azure.devcluster.openshift.com} + metadata: + name: ${CLUSTER_NAME} + controlPlane: + name: master + replicas: 3 + compute: + - name: worker + replicas: 3 + platform: + azure: + baseDomainResourceGroupName: os4-common + region: ${AZURE_REGION} + pullSecret: > + ${PULL_SECRET} + sshKey: | + ${SSH_PUB_KEY} + EOF else echo "Unsupported cluster type '${CLUSTER_NAME}'" exit 1 @@ -621,6 +652,14 @@ objects: value: ${NAMESPACE}-${JOB_NAME_HASH} - name: TYPE value: ${CLUSTER_TYPE} + - name: AWS_SHARED_CREDENTIALS_FILE + value: /etc/openshift-installer/.awscred + - name: AWS_REGION + value: us-east-1 + - name: AZURE_AUTH_LOCATION + value: /etc/openshift-installer/osServicePrincipal.json + - name: AZURE_REGION + value: centralus - name: KUBECONFIG value: /tmp/artifacts/installer/auth/kubeconfig command: @@ -760,7 +799,6 @@ objects: wait echo "Deprovisioning cluster ..." - export AWS_SHARED_CREDENTIALS_FILE=/etc/openshift-installer/.awscred openshift-install --dir /tmp/artifacts/installer destroy cluster } diff --git a/ci-operator/templates/openshift/installer/cluster-launch-installer-src.yaml b/ci-operator/templates/openshift/installer/cluster-launch-installer-src.yaml index 41c622c3d1d92..4c2f1ce8e9f89 100644 --- a/ci-operator/templates/openshift/installer/cluster-launch-installer-src.yaml +++ b/ci-operator/templates/openshift/installer/cluster-launch-installer-src.yaml @@ -28,8 +28,6 @@ parameters: - name: RESOURCES_TEST value: '{"requests": {"cpu": 1, "memory": "300Mi"}, "limits": {"memory": "3Gi"}}' - name: BASE_DOMAIN - value: origin-ci-int-aws.dev.rhcloud.com - required: true objects: @@ -98,6 +96,8 @@ objects: env: - name: AWS_SHARED_CREDENTIALS_FILE value: /tmp/cluster/.awscred + - name: AZURE_AUTH_LOCATION + value: /tmp/cluster/osServicePrincipal.json - name: ARTIFACT_DIR value: /tmp/artifacts - name: HOME @@ -157,6 +157,13 @@ objects: # TODO: make openshift-tests auto-discover this from cluster config export TEST_PROVIDER='{"type":"aws","region":"us-east-1","zone":"us-east-1a","multizone":true,"multimaster":true}' export KUBE_SSH_USER=core + elif [[ "${CLUSTER_TYPE}" == "azure" ]]; then + mkdir -p ~/.ssh + export PROVIDER_ARGS="-provider=azure -gce-zone=centralus" + # TODO: make openshift-tests auto-discover this from cluster config + export TEST_PROVIDER='{"type":"azure","region":"centralus","multizone":true,"multimaster":true}' + export KUBE_SSH_USER=core + fi elif [[ "${CLUSTER_TYPE}" == "openstack" ]]; then mkdir -p ~/.ssh cp /tmp/cluster/ssh-privatekey ~/.ssh/kube_openstack_rsa || true @@ -182,6 +189,10 @@ objects: value: /etc/openshift-installer/.awscred - name: AWS_REGION value: us-east-1 + - name: AZURE_AUTH_LOCATION + value: /etc/openshift-installer/osServicePrincipal.json + - name: AZURE_REGION + value: centralus - name: CLUSTER_NAME value: ${NAMESPACE}-${JOB_NAME_HASH} - name: BASE_DOMAIN @@ -225,8 +236,8 @@ objects: if [[ "${CLUSTER_TYPE}" == "aws" ]]; then cat > /tmp/artifacts/installer/install-config.yaml << EOF - apiVersion: v1beta4 - baseDomain: ${BASE_DOMAIN} + apiVersion: v1 + baseDomain: ${BASE_DOMAIN:-origin-ci-int-aws.dev.rhcloud.com} metadata: name: ${CLUSTER_NAME} controlPlane: @@ -255,10 +266,31 @@ objects: sshKey: | ${SSH_PUB_KEY} EOF + elif [[ "${CLUSTER_TYPE}" == "azure4" ]]; then + cat > /tmp/artifacts/installer/install-config.yaml << EOF + apiVersion: v1 + baseDomain: ${BASE_DOMAIN:-ci.azure.devcluster.openshift.com} + metadata: + name: ${CLUSTER_NAME} + controlPlane: + name: master + replicas: 3 + compute: + - name: worker + replicas: 3 + platform: + azure: + baseDomainResourceGroupName: os4-common + region: ${AZURE_REGION} + pullSecret: > + ${PULL_SECRET} + sshKey: | + ${SSH_PUB_KEY} + EOF elif [[ "${CLUSTER_TYPE}" == "openstack" ]]; then cat > /tmp/artifacts/installer/install-config.yaml << EOF - apiVersion: v1beta4 - baseDomain: ${BASE_DOMAIN} + apiVersion: v1 + baseDomain: ${BASE_DOMAIN:-origin-ci-int-aws.dev.rhcloud.com} metadata: name: ${CLUSTER_NAME} platform: @@ -296,6 +328,14 @@ objects: value: ${NAMESPACE}-${JOB_NAME_HASH} - name: TYPE value: ${CLUSTER_TYPE} + - name: AWS_SHARED_CREDENTIALS_FILE + value: /etc/openshift-installer/.awscred + - name: AWS_REGION + value: us-east-1 + - name: AZURE_AUTH_LOCATION + value: /etc/openshift-installer/osServicePrincipal.json + - name: AZURE_REGION + value: centralus - name: KUBECONFIG value: /tmp/artifacts/installer/auth/kubeconfig command: @@ -434,7 +474,6 @@ objects: wait echo "Deprovisioning cluster ..." - export AWS_SHARED_CREDENTIALS_FILE=/etc/openshift-installer/.awscred openshift-install --dir /tmp/artifacts/installer destroy cluster } diff --git a/cluster/ci/config/secret-mirroring/mapping.yaml b/cluster/ci/config/secret-mirroring/mapping.yaml index f4a9e09930b9b..95f86bb3bedeb 100644 --- a/cluster/ci/config/secret-mirroring/mapping.yaml +++ b/cluster/ci/config/secret-mirroring/mapping.yaml @@ -17,6 +17,12 @@ secrets: to: namespace: ci-stg name: cluster-secrets-azure +- from: + namespace: ci + name: cluster-secrets-azure4 + to: + namespace: ci-stg + name: cluster-secrets-azure4 - from: namespace: ci name: cluster-secrets-gcp diff --git a/cluster/test-deploy/azure4/.gitignore b/cluster/test-deploy/azure4/.gitignore new file mode 100644 index 0000000000000..cec2e8a226c09 --- /dev/null +++ b/cluster/test-deploy/azure4/.gitignore @@ -0,0 +1,5 @@ +* +!.type +!.gitignore +!OWNERS +!secret_example diff --git a/cluster/test-deploy/azure4/.type b/cluster/test-deploy/azure4/.type new file mode 100644 index 0000000000000..0a6f56f0921f9 --- /dev/null +++ b/cluster/test-deploy/azure4/.type @@ -0,0 +1 @@ +azure4 diff --git a/cluster/test-deploy/azure4/secret_example b/cluster/test-deploy/azure4/secret_example new file mode 100644 index 0000000000000..9153479a2f856 --- /dev/null +++ b/cluster/test-deploy/azure4/secret_example @@ -0,0 +1,6 @@ +{ + "subscriptionId": "xxxxx", + "clientId": "xxxxx", + "clientSecret": "xxxxx", + "tenantId": "xxxxx" +}