diff --git a/ci-operator/jobs/openshift/openshift-azure/openshift-openshift-azure-presubmits.yaml b/ci-operator/jobs/openshift/openshift-azure/openshift-openshift-azure-presubmits.yaml index a8d47badb31ae..d4130b46847c3 100644 --- a/ci-operator/jobs/openshift/openshift-azure/openshift-openshift-azure-presubmits.yaml +++ b/ci-operator/jobs/openshift/openshift-azure/openshift-openshift-azure-presubmits.yaml @@ -178,3 +178,63 @@ presubmits: - secret: name: cluster-secrets-azure trigger: ((?m)^/test upgrade,?(\s+|$)) + - agent: kubernetes + always_run: false + branches: + - master + context: ci/prow/conformance + decorate: true + labels: + artifacts: images + name: pull-ci-openshift-azure-conformance + rerun_command: /test conformance + skip_cloning: true + spec: + containers: + - args: + - --artifact-dir=$(ARTIFACTS) + - --give-pr-author-access-to-namespace=true + - --secret-dir=/usr/local/e2e-azure-secret + - --template=/usr/local/e2e-azure-conformance + - --target=[output:stable:sync] + - --target=e2e-azure-conformance + command: + - ci-operator + env: + - name: TEST_FOCUS + value: Suite:openshift/conformance/parallel + - name: TEST_FOCUS_SERIAL + value: Suite:openshift/registry/serial + - name: CONFIG_SPEC + valueFrom: + configMapKeyRef: + key: master.yaml + name: ci-operator-openshift-openshift-azure + - name: JOB_NAME_SAFE + value: e2e-azure-conformance + - name: CLUSTER_TYPE + value: azure + image: ci-operator:latest + name: test + resources: + limits: + cpu: 500m + requests: + cpu: 10m + volumeMounts: + - mountPath: /usr/local/e2e-azure-conformance + name: job-definition + subPath: cluster-launch-e2e-azure-conformance.yaml + - mountPath: /usr/local/e2e-azure-secret + name: cluster-secrets-azure + serviceAccountName: ci-operator + volumes: + - configMap: + name: prow-job-cluster-launch-e2e-azure-conformance + name: job-definition + - name: cluster-secrets-azure + projected: + sources: + - secret: + name: cluster-secrets-azure + trigger: ((?m)^/test conformance,?(\s+|$)) diff --git a/ci-operator/templates/openshift-azure/README.md b/ci-operator/templates/openshift-azure/README.md new file mode 100644 index 0000000000000..89c20f8f04f80 --- /dev/null +++ b/ci-operator/templates/openshift-azure/README.md @@ -0,0 +1,2 @@ +# These templates are being used for testing OpenShift on Azure. +# At the moment it cannot be reused to launch cluster for Origin tests diff --git a/ci-operator/templates/openshift-azure/cluster-launch-e2e-azure-conformance.yaml b/ci-operator/templates/openshift-azure/cluster-launch-e2e-azure-conformance.yaml new file mode 100644 index 0000000000000..f0682df8a823d --- /dev/null +++ b/ci-operator/templates/openshift-azure/cluster-launch-e2e-azure-conformance.yaml @@ -0,0 +1,242 @@ +kind: Template +apiVersion: template.openshift.io/v1 + +parameters: +- name: JOB_NAME_SAFE + required: true +- name: JOB_NAME_HASH + required: true +- name: LOCAL_IMAGE_BIN + required: true +- name: IMAGE_TESTS + required: true +- name: IMAGE_SYNC + required: true +- name: NAMESPACE + required: true +- name: CLUSTER_TYPE + value: "azure" + required: true +- name: TEST_FOCUS +- name: TEST_SKIP + value: "\\[local\\]" +- name: TEST_FOCUS_SERIAL +- name: TEST_SKIP_SERIAL + value: "\\[local\\]" + +objects: + +# We want the cluster to be able to access these images +- kind: RoleBinding + apiVersion: authorization.openshift.io/v1 + metadata: + name: ${JOB_NAME_SAFE}-image-puller + namespace: ${NAMESPACE} + roleRef: + name: system:image-puller + subjects: + - kind: SystemGroup + name: system:unauthenticated + +- kind: Pod + apiVersion: v1 + metadata: + name: ${JOB_NAME_SAFE} + namespace: ${NAMESPACE} + annotations: + # we want to gather the teardown logs no matter what + ci-operator.openshift.io/wait-for-container-artifacts: teardown + spec: + restartPolicy: Never + activeDeadlineSeconds: 10800 + terminationGracePeriodSeconds: 600 + volumes: + - name: artifacts + emptyDir: {} + - name: shared-tmp + emptyDir: {} + - name: openshift-tmp + emptyDir: {} + - name: cluster-secrets-azure + secret: + secretName: e2e-azure-secret + + containers: + + # Executes origin conformance tests + - name: test + image: ${IMAGE_TESTS} + resources: + requests: + cpu: 1 + memory: 300Mi + limits: + cpu: 3 + memory: 4Gi + volumeMounts: + - name: shared-tmp + mountPath: /tmp/shared + - name: openshift-tmp + mountPath: /tmp/openshift + - name: artifacts + mountPath: /tmp/artifacts + env: + - name: HOME + value: /tmp/shared/home + command: + - /bin/bash + - -c + - | + #!/bin/bash + set -euo pipefail + + trap 'touch /tmp/shared/exit' EXIT + trap 'kill $(jobs -p); exit 0' TERM + + cp "$(which oc)" /tmp/shared/ + + mkdir -p "${HOME}" + + # wait until the setup job creates admin.kubeconfig + while true; do + if [[ ! -f /tmp/shared/_data/_out/admin.kubeconfig ]]; then + sleep 15 & wait + continue + fi + # if we got admin kubeconfig file with failure marker, ignore. Teardown is in progress. + if [[ -f /tmp/shared/exit ]]; then + exit 1 + fi + break + done + echo "Found shared kubeconfig" + + # don't let clients impact the global kubeconfig + cp -r /tmp/shared/_data /tmp/openshift/ + export KUBECONFIG=/tmp/openshift/_data/_out/admin.kubeconfig + + PATH=/usr/libexec/origin:$PATH + + # TODO: the test binary should really be a more structured command - most of these flags should be + # autodetected from the running cluster. + # TODO: bump nodes up to 40 again + set -x + if [[ -n "${TEST_FOCUS}" ]]; then + ginkgo -v -noColor -nodes=30 $( which extended.test ) -- \ + -ginkgo.focus="${TEST_FOCUS}" -ginkgo.skip="${TEST_SKIP}" \ + -e2e-output-dir /tmp/artifacts -report-dir /tmp/artifacts/junit \ + -test.timeout=2h || rc=$? + fi + if [[ -n "${TEST_FOCUS_SERIAL}" ]]; then + ginkgo -v -noColor -nodes=1 $( which extended.test ) -- \ + -ginkgo.focus="${TEST_FOCUS_SERIAL}" -ginkgo.skip="${TEST_SKIP_SERIAL}" \ + -e2e-output-dir /tmp/artifacts -report-dir /tmp/artifacts/junit/serial \ + -test.timeout=2h || rc=$? + fi + exit ${rc:-0} + + # Runs an install + - name: setup + image: ${LOCAL_IMAGE_BIN} + volumeMounts: + - name: shared-tmp + mountPath: /tmp/shared + - name: cluster-secrets-azure + mountPath: /etc/azure/credentials + env: + - name: INSTANCE_PREFIX + value: ${NAMESPACE}-${JOB_NAME_HASH} + - name: TYPE + value: ${CLUSTER_TYPE} + - name: HOME + value: /tmp/shared/home + - name: SYNC_IMAGE + value: ${IMAGE_SYNC} + command: + - /bin/bash + - -c + - | + #!/bin/bash + set -euo pipefail + + # trap acts as a switch/router for the next phases + trap 'rc=$?; if [[ $rc -ne 0 ]]; then + touch /tmp/shared/exit; + fi; + cp -r /go/src/github.com/openshift/openshift-azure/_data /tmp/shared &>/dev/null + exit $rc' EXIT + trap 'kill $(jobs -p); exit 0' TERM + + # Cluster creation specific configuration. + mkdir -p "${HOME}" + source /etc/azure/credentials/secret + az login --service-principal -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} --tenant ${AZURE_TENANT_ID} &>/dev/null + # aad integration configuration - we dont test aad so populate as dummy + export AZURE_AAD_CLIENT_ID=$AZURE_CLIENT_ID + export AZURE_AAD_CLIENT_SECRET=$AZURE_CLIENT_SECRET + echo "Using sync image ${SYNC_IMAGE}" + export DNS_DOMAIN=osadev.cloud + export DNS_RESOURCEGROUP=dns + export DEPLOY_VERSION=v3.10 + export RUN_SYNC_LOCAL=true + export IMAGE_RESOURCEGROUP=images + export IMAGE_RESOURCENAME=$(az image list -g $IMAGE_RESOURCEGROUP -o json --query "[?starts_with(name, '${DEPLOY_OS:-rhel7}-${DEPLOY_VERSION//v}') && tags.valid=='true'].name | sort(@) | [-1]" | tr -d '"') + # create cluster for test + cd /go/src/github.com/openshift/openshift-azure/ + ./hack/create.sh ${INSTANCE_PREFIX} + + # Performs cleanup of all created resources + - name: teardown + image: ${LOCAL_IMAGE_BIN} + volumeMounts: + - name: shared-tmp + mountPath: /tmp/shared + - name: cluster-secrets-azure + mountPath: /etc/azure/credentials + - name: artifacts + mountPath: /tmp/artifacts + env: + - name: INSTANCE_PREFIX + value: ${NAMESPACE}-${JOB_NAME_HASH} + - name: TYPE + value: ${CLUSTER_TYPE} + - name: HOME + value: /tmp/shared/home + command: + - /bin/bash + - -c + - | + #!/bin/bash + + # teardown is collecting debug data and deleting all used resources + function teardown() { + set +e + mkdir -p "${HOME}" + export HOME=/tmp/shared + export DNS_DOMAIN=osadev.cloud + export DNS_RESOURCEGROUP=dns + export KUBECONFIG=/tmp/shared/_data/_out/admin.kubeconfig + + cp -r /tmp/shared/_data /go/src/github.com/openshift/openshift-azure/ + cd /go/src/github.com/openshift/openshift-azure/ + source /etc/azure/credentials/secret + az login --service-principal -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} --tenant ${AZURE_TENANT_ID} &>/dev/null + oc get po --all-namespaces -o wide > /tmp/artifacts/pods + oc get no -o wide > /tmp/artifacts/nodes + oc get events --all-namespaces > /tmp/artifacts/events + ./hack/delete.sh ${INSTANCE_PREFIX} + + trap 'teardown' EXIT + trap 'kill $(jobs -p); exit 0' TERM + } + + trap 'teardown' EXIT + trap 'kill $(jobs -p); exit 0' TERM + + # teardown is triggered on file marker + for i in `seq 1 120`; do + if [[ -f /tmp/shared/exit ]]; then + exit 0 + fi + sleep 60 & wait + done diff --git a/ci-operator/templates/openshift-azure/cluster-launch-e2e-azure-upgrade.yaml b/ci-operator/templates/openshift-azure/cluster-launch-e2e-azure-upgrade.yaml index b3027067fd9ae..9c4488e036b2a 100644 --- a/ci-operator/templates/openshift-azure/cluster-launch-e2e-azure-upgrade.yaml +++ b/ci-operator/templates/openshift-azure/cluster-launch-e2e-azure-upgrade.yaml @@ -1,5 +1,3 @@ -# This template is being used for testing OpenShift on Azure. -# At the moment it cannot be reused to launch cluster for Origin tests kind: Template apiVersion: template.openshift.io/v1 @@ -32,7 +30,6 @@ objects: - kind: SystemGroup name: system:unauthenticated -# The e2e pod spins up a cluster, runs e2e tests, and then cleans up the cluster. - kind: Pod apiVersion: v1 metadata: @@ -58,7 +55,7 @@ objects: containers: - # Runs an upgrade + # Runs an upgrade test - name: test image: ${LOCAL_IMAGE_BIN} resources: diff --git a/ci-operator/templates/openshift-azure/cluster-launch-e2e-azure.yaml b/ci-operator/templates/openshift-azure/cluster-launch-e2e-azure.yaml index b8b822c43b907..edda7496c9b47 100644 --- a/ci-operator/templates/openshift-azure/cluster-launch-e2e-azure.yaml +++ b/ci-operator/templates/openshift-azure/cluster-launch-e2e-azure.yaml @@ -1,5 +1,3 @@ -# This template is being used for testing OpenShift on Azure. -# At the moment it cannot be reused to launch cluster for Origin tests kind: Template apiVersion: template.openshift.io/v1 @@ -32,7 +30,6 @@ objects: - kind: SystemGroup name: system:unauthenticated -# The e2e pod spins up a cluster, runs e2e tests, and then cleans up the cluster. - kind: Pod apiVersion: v1 metadata: diff --git a/cluster/ci/config/prow/plugins.yaml b/cluster/ci/config/prow/plugins.yaml index 4f1b498388628..c5d1a49ae68ba 100644 --- a/cluster/ci/config/prow/plugins.yaml +++ b/cluster/ci/config/prow/plugins.yaml @@ -136,6 +136,8 @@ config_updater: name: prow-job-cluster-launch-e2e-azure ci-operator/templates/openshift-azure/cluster-launch-e2e-azure-upgrade.yaml: name: prow-job-cluster-launch-e2e-azure-upgrade + ci-operator/templates/openshift-azure/cluster-launch-e2e-azure-conformance.yaml: + name: prow-job-cluster-launch-e2e-azure-conformance ci-operator/templates/cluster-launch-e2e-gmontero-testing.yaml: name: prow-job-cluster-launch-e2e-gmontero-testing ci-operator/templates/cluster-launch-e2e-upgrade.yaml: diff --git a/projects/azure/README.md b/projects/azure/README.md index 36ddf4c597883..3e30be9f22c4a 100644 --- a/projects/azure/README.md +++ b/projects/azure/README.md @@ -61,4 +61,4 @@ oc get secret cluster-secrets-azure-temp --export -n azure -o yaml | sed 's/clus ``` source ./cluster/test-deploy/azure/secret oc create secret generic cluster-secrets-azure --from-literal=azure_client_id=${AZURE_CLIENT_ID} --from-literal=azure_client_secret=${AZURE_CLIENT_SECRET} --from-literal=azure_tenant_id=${AZURE_TENANT_ID} --from-literal=azure_subscription_id=${AZURE_SUBSCRIPTION_ID} -o yaml --dry-run | oc apply -n azure -f - -``` \ No newline at end of file +```