diff --git a/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.11.yaml b/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.11.yaml index 4a8686977ee99..35f63e6db2f1e 100644 --- a/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.11.yaml +++ b/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.11.yaml @@ -130,23 +130,24 @@ resources: memory: 200Mi tests: - as: ocp-e2e-aws-heterogeneous - interval: 72h + cron: 33 7 * * 3 steps: cluster_profile: aws-2 dependencies: OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest env: BASE_DOMAIN: aws-2.ci.openshift.org - workflow: openshift-e2e-aws + workflow: openshift-e2e-aws-heterogeneous - as: ocp-e2e-serial-aws-heterogeneous - interval: 72h + cron: 51 15 * * 4 steps: cluster_profile: aws-2 dependencies: OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest env: BASE_DOMAIN: aws-2.ci.openshift.org - workflow: openshift-e2e-aws-serial + TEST_SUITE: openshift/conformance/serial + workflow: openshift-e2e-aws-heterogeneous - as: ocp-e2e-aws-arm64 cron: 58 16 */1 * 1-5 steps: diff --git a/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.12.yaml b/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.12.yaml index f35b21994d3da..71e4f45e3a0a0 100644 --- a/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.12.yaml +++ b/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.12.yaml @@ -130,23 +130,24 @@ resources: memory: 200Mi tests: - as: ocp-e2e-aws-ovn-heterogeneous - interval: 72h + cron: 08 11 * * 5 steps: cluster_profile: aws-2 dependencies: OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest env: BASE_DOMAIN: aws-2.ci.openshift.org - workflow: openshift-e2e-aws + workflow: openshift-e2e-aws-heterogeneous - as: ocp-e2e-serial-aws-ovn-heterogeneous - interval: 72h + cron: 19 23 * * 2 steps: cluster_profile: aws-2 dependencies: OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest env: BASE_DOMAIN: aws-2.ci.openshift.org - workflow: openshift-e2e-aws-serial + TEST_SUITE: openshift/conformance/serial + workflow: openshift-e2e-aws-heterogeneous - as: ocp-e2e-aws-sdn-arm64 interval: 72h steps: diff --git a/ci-operator/jobs/openshift/multiarch/openshift-multiarch-master-periodics.yaml b/ci-operator/jobs/openshift/multiarch/openshift-multiarch-master-periodics.yaml index dc29c3a42c2e2..291aad1770948 100644 --- a/ci-operator/jobs/openshift/multiarch/openshift-multiarch-master-periodics.yaml +++ b/ci-operator/jobs/openshift/multiarch/openshift-multiarch-master-periodics.yaml @@ -1888,6 +1888,7 @@ periodics: secretName: result-aggregator - agent: kubernetes cluster: build05 + cron: 33 7 * * 3 decorate: true decoration_config: skip_cloning: true @@ -1895,7 +1896,6 @@ periodics: - base_ref: master org: openshift repo: multiarch - interval: 72h labels: ci-operator.openshift.io/cloud: aws ci-operator.openshift.io/cloud-cluster-profile: aws-2 @@ -2642,6 +2642,7 @@ periodics: secretName: result-aggregator - agent: kubernetes cluster: build05 + cron: 51 15 * * 4 decorate: true decoration_config: skip_cloning: true @@ -2649,7 +2650,6 @@ periodics: - base_ref: master org: openshift repo: multiarch - interval: 72h labels: ci-operator.openshift.io/cloud: aws ci-operator.openshift.io/cloud-cluster-profile: aws-2 @@ -4075,6 +4075,7 @@ periodics: secretName: result-aggregator - agent: kubernetes cluster: build05 + cron: 08 11 * * 5 decorate: true decoration_config: skip_cloning: true @@ -4082,7 +4083,6 @@ periodics: - base_ref: master org: openshift repo: multiarch - interval: 72h labels: ci-operator.openshift.io/cloud: aws ci-operator.openshift.io/cloud-cluster-profile: aws-2 @@ -4830,6 +4830,7 @@ periodics: secretName: result-aggregator - agent: kubernetes cluster: build05 + cron: 19 23 * * 2 decorate: true decoration_config: skip_cloning: true @@ -4837,7 +4838,6 @@ periodics: - base_ref: master org: openshift repo: multiarch - interval: 72h labels: ci-operator.openshift.io/cloud: aws ci-operator.openshift.io/cloud-cluster-profile: aws-2 diff --git a/ci-operator/step-registry/ipi/install/heterogeneous/OWNERS b/ci-operator/step-registry/ipi/install/heterogeneous/OWNERS new file mode 100644 index 0000000000000..2a3ace342adc0 --- /dev/null +++ b/ci-operator/step-registry/ipi/install/heterogeneous/OWNERS @@ -0,0 +1,6 @@ +approvers: +- aleskandro +- deepsm007 +- jaypoulz +- jeffdyoung +- Prashanth684 \ No newline at end of file diff --git a/ci-operator/step-registry/ipi/install/heterogeneous/ipi-install-heterogeneous-commands.sh b/ci-operator/step-registry/ipi/install/heterogeneous/ipi-install-heterogeneous-commands.sh new file mode 100644 index 0000000000000..0ce5a5283181c --- /dev/null +++ b/ci-operator/step-registry/ipi/install/heterogeneous/ipi-install-heterogeneous-commands.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +export KUBECONFIG=${SHARED_DIR}/kubeconfig + +trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM + +# Make sure yq is installed +if [ ! -f /tmp/yq-v4 ]; then + # TODO move to image + curl -L "https://github.com/mikefarah/yq/releases/download/v4.25.3/yq_linux_$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/)" -o /tmp/yq-v4 && chmod +x /tmp/yq-v4 + PATH=${PATH}:/tmp +fi + +# Make sure jq is installed +if ! command -v jq; then + # TODO move to image + curl -sL https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 > /tmp/jq + chmod +x /tmp/jq + PATH=${PATH}:/tmp +fi + +dir=/tmp/installer +mkdir "${dir}/" + +REGION=${LEASED_RESOURCE} + +echo "Fetching Worker MachineSet..." +oc -n openshift-machine-api get -o json machinesets | jq '[.items[] | select(.spec.template.metadata.labels["machine.openshift.io/cluster-api-machine-role"] == "worker")][0]' > ${dir}/99_openshift-cluster-api_worker-machineset-0.yaml + +# AMI for AWS ARM +echo "Extracting AMI..." +oc -n openshift-machine-config-operator get configmap/coreos-bootimages -oyaml > ${dir}/coreos-bootimages.yaml +yq-v4 eval ".data.stream" ${dir}/coreos-bootimages.yaml > ${dir}/machineset.yaml +amiid_workers_additional=$(yq-v4 ".architectures.${ADDITIONAL_WORKER_ARCHITECTURE}.images.aws.regions.\"${REGION}\".image" ${dir}/machineset.yaml) +echo "Updating the machineset with ${ADDITIONAL_WORKER_VM_TYPE} and ami ${amiid_workers_additional} ..." +yq-v4 eval ".metadata.name += \"-additional\" +| .spec.replicas = ${ADDITIONAL_WORKERS} +| .spec.selector.matchLabels.\"machine.openshift.io/cluster-api-machineset\" = .metadata.name +| .spec.template.metadata.labels.\"machine.openshift.io/cluster-api-machineset\" = .metadata.name" \ +${dir}/99_openshift-cluster-api_worker-machineset-0.yaml > ${dir}/99_openshift-cluster-api_worker-machineset-additional.yaml + +yq-v4 eval ".spec.template.spec.providerSpec.value.ami.id = \"${amiid_workers_additional}\" + | .spec.template.spec.providerSpec.value.instanceType = \"${ADDITIONAL_WORKER_VM_TYPE}\" + " -i ${dir}/99_openshift-cluster-api_worker-machineset-additional.yaml + +echo "Creating ${ADDITIONAL_WORKER_ARCHITECTURE} worker MachineSet" +exec oc create -f ${dir}/99_openshift-cluster-api_worker-machineset-additional.yaml & + +wait "$!" +ret="$?" + +if test "${ret}" -eq 0 ; then + touch "${SHARED_DIR}/success" + # Save console URL in `console.url` file so that ci-chat-bot could report success + echo "https://$(env KUBECONFIG=${dir}/auth/kubeconfig oc -n openshift-console get routes console -o=jsonpath='{.spec.host}')" > "${SHARED_DIR}/console.url" +fi + +echo "Exiting with ret=${ret}" +exit "${ret}" diff --git a/ci-operator/step-registry/ipi/install/heterogeneous/ipi-install-heterogeneous-ref.metadata.json b/ci-operator/step-registry/ipi/install/heterogeneous/ipi-install-heterogeneous-ref.metadata.json new file mode 100644 index 0000000000000..9ec662ed79ca2 --- /dev/null +++ b/ci-operator/step-registry/ipi/install/heterogeneous/ipi-install-heterogeneous-ref.metadata.json @@ -0,0 +1,12 @@ +{ + "path": "ipi/install/heterogeneous/ipi-install-heterogeneous-ref.yaml", + "owners": { + "approvers": [ + "aleskandro", + "deepsm007", + "jaypoulz", + "jeffdyoung", + "Prashanth684" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/ipi/install/heterogeneous/ipi-install-heterogeneous-ref.yaml b/ci-operator/step-registry/ipi/install/heterogeneous/ipi-install-heterogeneous-ref.yaml new file mode 100644 index 0000000000000..7370f612e7875 --- /dev/null +++ b/ci-operator/step-registry/ipi/install/heterogeneous/ipi-install-heterogeneous-ref.yaml @@ -0,0 +1,21 @@ +ref: + as: ipi-install-heterogeneous + from: tests + grace_period: 10m + commands: ipi-install-heterogeneous-commands.sh + resources: + requests: + cpu: 10m + memory: 100Mi + env: + - name: ADDITIONAL_WORKERS + default: "2" + documentation: "Worker count for heterogeneous deploy" + - name: ADDITIONAL_WORKER_ARCHITECTURE + default: "aarch64" + documentation: "Heterogeneous worker architecture, default set to arm64(aarch64)" + - name: ADDITIONAL_WORKER_VM_TYPE + default: "m6g.xlarge" + documentation: "VM Instance type for Heterogeneous worker, default set to arm64 instance type" + documentation: |- + The heterogeneous custom step adds additional worker node post configuration for adding custom arch worker node. diff --git a/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/OWNERS b/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/OWNERS new file mode 100644 index 0000000000000..2a3ace342adc0 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/OWNERS @@ -0,0 +1,6 @@ +approvers: +- aleskandro +- deepsm007 +- jaypoulz +- jeffdyoung +- Prashanth684 \ No newline at end of file diff --git a/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/openshift-e2e-aws-heterogeneous-workflow.metadata.json b/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/openshift-e2e-aws-heterogeneous-workflow.metadata.json new file mode 100644 index 0000000000000..7d81ff3a846d2 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/openshift-e2e-aws-heterogeneous-workflow.metadata.json @@ -0,0 +1,12 @@ +{ + "path": "openshift/e2e/aws/heterogeneous/openshift-e2e-aws-heterogeneous-workflow.yaml", + "owners": { + "approvers": [ + "aleskandro", + "deepsm007", + "jaypoulz", + "jeffdyoung", + "Prashanth684" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/openshift-e2e-aws-heterogeneous-workflow.yaml b/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/openshift-e2e-aws-heterogeneous-workflow.yaml new file mode 100644 index 0000000000000..fbf04ee68cf7d --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/openshift-e2e-aws-heterogeneous-workflow.yaml @@ -0,0 +1,39 @@ +workflow: + as: openshift-e2e-aws-heterogeneous + steps: + allow_best_effort_post_steps: true + pre: + - chain: ipi-aws-pre + - ref: ipi-install-heterogeneous + test: + - ref: openshift-e2e-test + post: + - chain: gather-core-dump + - chain: ipi-aws-post + env: + TEST_SKIPS: >- + The HAProxy router should\| deploymentconfigs\| Users should be able to create and run a job in a user project\| + Managed cluster should should expose cluster services outside the cluster\| FIPS TestFIPS\| + TestPodDefaultCapabilities\| Multi-stage image builds should succeed\| Optimized image builds should succeed\| + build can reference a cluster service with a build being created from new-build should be able to run a build that references a cluster service\| + custom build with buildah being created from new-build should complete build with custom builder image\| + oc new-app should succeed with a --name of 58 characters\| oc can run inside of a busybox container\| + oc debug deployment configs from a build\| oc rsh specific flags should work well when access to a remote shell\| + prune builds based on settings in the buildconfig should prune completed builds based on the successfulBuildsHistoryLimit setting\| + s2i build with a quota Building from a template should create an s2i build with a quota and run it\| + s2i build with a root user image should create a root build and pass with a privileged SCC\| + verify /run filesystem contents are writeable using a simple Docker Strategy Build\| + verify /run filesystem contents do not have unexpected content using a simple Docker Strategy Build\| + Pods cannot access the /config/master API endpoint\| Image append should create images by appending them\| + Image extract should extract content from an image\| Image info should display information about images\| + Image layer subresource should return layers from tagged images\| oc tag should change image reference for internal images\| + Prometheus when installed on the cluster should provide ingress metrics\| + Prometheus when installed on the cluster should start and expose a secured proxy and unsecured metrics\| + OpenShift alerting rules should have a runbook_url annotation if the alert is critical\| + OpenShift alerting rules should have a valid severity label\| OpenShift alerting rules should have description and summary annotations\| + Prometheus when installed on the cluster should start and expose a secured proxy and verify build metrics\| + egressFirewall should have no impact outside its namespace\| should ensure ipv4 egressrouter cni resources are created\| + pod should start with all sysctl on whitelist\| pod sysctls should not affect node\| + when FIPS is disabled the HAProxy router should serve routes when configured with a 1024-bit RSA key + documentation: |- + The Openshift E2E HETEROGENEOUS AWS workflow executes the common end-to-end test suite on AWS with a default cluster configuration.