diff --git a/ci-operator/config/openshift/installer/openshift-installer-master.yaml b/ci-operator/config/openshift/installer/openshift-installer-master.yaml index f5e034873ced4..032479b604eac 100644 --- a/ci-operator/config/openshift/installer/openshift-installer-master.yaml +++ b/ci-operator/config/openshift/installer/openshift-installer-master.yaml @@ -209,6 +209,12 @@ releases: include_built_images: true name: "4.17" namespace: ocp + multi-latest: + candidate: + architecture: multi + product: ocp + stream: nightly + version: "4.17" resources: '*': requests: @@ -869,6 +875,20 @@ tests: PLATFORM_EXTERNAL_CCM_ENABLED: "yes" workflow: openshift-e2e-external-aws timeout: 6h0m0s +- always_run: false + as: e2e-aws-ovn-heterogeneous + optional: true + steps: + cluster_profile: aws + dependencies: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest + env: + COMPUTE_ARCH: amd64 + CONTROL_ARCH: arm64 + FEATURE_GATES: '["MultiArchInstallAWS=true"]' + FEATURE_SET: CustomNoUpgrade + workflow: openshift-e2e-aws-heterogeneous-day-0 + timeout: 6h0m0s zz_generated_metadata: branch: master org: openshift diff --git a/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.17.yaml b/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.17.yaml index 68e2a4bf544dd..0f3ef8691228e 100644 --- a/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.17.yaml +++ b/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.17.yaml @@ -211,6 +211,28 @@ tests: pod should start\| pod sysctls\| build volumes should mount given secrets and configmaps into the build pod workflow: openshift-e2e-aws-heterogeneous +- as: ocp-e2e-aws-ovn-heterogeneous-day-0 + cron: 08 11 * * 5 + steps: + cluster_profile: aws-2 + dependencies: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest + env: + BASE_DOMAIN: aws-2.ci.openshift.org + COMPUTE_ARCH: amd64 + CONTROL_ARCH: arm64 + TEST_SKIPS: deploymentconfigs\| should expose cluster services outside the cluster\| + FIPS TestFIPS\| Multi-stage image builds should succeed\| Optimized image + builds should succeed\| build can reference a cluster service\| custom build + with buildah\| oc new-app should succeed\| prune builds based on settings\| + s2i build with a root\| verify /run filesystem contents\| oc can run\| oc + debug\| oc idle\| Pods cannot access\| Image append should create\| Image + extract should extract\| Image info should display\| Image layer subresource\| + oc tag should change image\| when installed on the cluster should\| OpenShift + alerting rules\| The HAProxy router should\| egressrouter cni resources\| + pod should start\| pod sysctls\| build volumes should mount given secrets + and configmaps into the build pod + workflow: openshift-e2e-aws-heterogeneous-day-0 - as: ocp-e2e-serial-aws-ovn-heterogeneous cron: 19 23 * * 2 steps: diff --git a/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml b/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml index 22eadcea14332..714838ebd1b7c 100644 --- a/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml +++ b/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml @@ -3907,6 +3907,88 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-aws-ovn-fips,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^master$ + - ^master- + cluster: build03 + context: ci/prow/e2e-aws-ovn-heterogeneous + decorate: true + decoration_config: + timeout: 6h0m0s + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-installer-master-e2e-aws-ovn-heterogeneous + optional: true + rerun_command: /test e2e-aws-ovn-heterogeneous + skip_if_only_changed: (^(docs|upi|hack)/)|((^|/)OWNERS(_ALIASES)?$)|((^|/)[A-Z]+\.md$)|(^images/(agent|alibabacloud|azure|azurestack|baremetal|gcp|ibmcloud|libvirt|nutanix|openstack|ovirt|powervs|vsphere)/) + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --secret-dir=/usr/local/e2e-aws-ovn-heterogeneous-cluster-profile + - --target=e2e-aws-ovn-heterogeneous + command: + - ci-operator + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /usr/local/e2e-aws-ovn-heterogeneous-cluster-profile + name: cluster-profile + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: cluster-profile + secret: + secretName: cluster-secrets-aws + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-aws-ovn-heterogeneous,?($|\s.*) - agent: kubernetes always_run: false branches: diff --git a/ci-operator/jobs/openshift/multiarch/openshift-multiarch-master-periodics.yaml b/ci-operator/jobs/openshift/multiarch/openshift-multiarch-master-periodics.yaml index 7becc2a57593e..5192e7e18cbbd 100644 --- a/ci-operator/jobs/openshift/multiarch/openshift-multiarch-master-periodics.yaml +++ b/ci-operator/jobs/openshift/multiarch/openshift-multiarch-master-periodics.yaml @@ -23835,6 +23835,87 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build05 + cron: 08 11 * * 5 + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: master + org: openshift + repo: multiarch + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: aws-2 + ci-operator.openshift.io/variant: nightly-4.17 + ci.openshift.io/generator: prowgen + job-release: "4.17" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-multiarch-master-nightly-4.17-ocp-e2e-aws-ovn-heterogeneous-day-0 + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --secret-dir=/usr/local/ocp-e2e-aws-ovn-heterogeneous-day-0-cluster-profile + - --target=ocp-e2e-aws-ovn-heterogeneous-day-0 + - --variant=nightly-4.17 + command: + - ci-operator + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /usr/local/ocp-e2e-aws-ovn-heterogeneous-day-0-cluster-profile + name: cluster-profile + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: cluster-profile + secret: + secretName: cluster-secrets-aws-2 + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build05 decorate: true diff --git a/ci-operator/step-registry/ipi/conf/aws/ipi-conf-aws-commands.sh b/ci-operator/step-registry/ipi/conf/aws/ipi-conf-aws-commands.sh index c1d14663760c1..d0f165c20f43b 100755 --- a/ci-operator/step-registry/ipi/conf/aws/ipi-conf-aws-commands.sh +++ b/ci-operator/step-registry/ipi/conf/aws/ipi-conf-aws-commands.sh @@ -63,10 +63,6 @@ function eval_instance_capacity() { set -o errexit } - -# BootstrapInstanceType gets its value from pkg/types/aws/defaults/platform.go -architecture=${OCP_ARCH:-"amd64"} - CONTROL_PLANE_INSTANCE_SIZE="xlarge" if [[ "${SIZE_VARIANT}" == "xlarge" ]]; then CONTROL_PLANE_INSTANCE_SIZE="8xlarge" @@ -91,7 +87,9 @@ if [[ "${CLUSTER_TYPE}" =~ ^aws-s?c2s$ ]]; then fi elif [[ "${CLUSTER_TYPE}" == "aws-arm64" ]] || [[ "${OCP_ARCH}" == "arm64" ]]; then # ARM 64 - architecture="arm64" + CONTROL_ARCH="arm64" + COMPUTE_ARCH="arm64" + if [[ "${COMPUTE_NODE_TYPE}" == "" ]]; then COMPUTE_NODE_TYPE="m6g.xlarge" fi @@ -100,7 +98,7 @@ elif [[ "${CLUSTER_TYPE}" == "aws-arm64" ]] || [[ "${OCP_ARCH}" == "arm64" ]]; t CONTROL_PLANE_INSTANCE_TYPE="m6g.${CONTROL_PLANE_INSTANCE_SIZE}" fi else - # AMD 64 + # AMD 64 or Multiarch Compute # m6a (AMD) are more cost effective than other x86 instance types # for general purpose work. Use by default, when supported in the @@ -113,20 +111,34 @@ else # Do not change auto-types unless it is coordinated with the cloud # financial operations team. Savings plans may be in place to # decrease the cost of certain instance families. - if [[ "${CONTROL_PLANE_INSTANCE_TYPE}" == "" ]]; then - if [[ "${IS_M6A_REGION}" == "yes" ]]; then - CONTROL_PLANE_INSTANCE_TYPE=$(eval_instance_capacity "m6a.${CONTROL_PLANE_INSTANCE_SIZE}" "m6i.${CONTROL_PLANE_INSTANCE_SIZE}") - else - CONTROL_PLANE_INSTANCE_TYPE="m6i.${CONTROL_PLANE_INSTANCE_SIZE}" + if [[ "${CONTROL_ARCH}" == "amd64" ]]; then + if [[ "${CONTROL_PLANE_INSTANCE_TYPE}" == "" ]]; then + if [[ "${IS_M6A_REGION}" == "yes" ]]; then + CONTROL_PLANE_INSTANCE_TYPE=$(eval_instance_capacity "m6a.${CONTROL_PLANE_INSTANCE_SIZE}" "m6i.${CONTROL_PLANE_INSTANCE_SIZE}") + else + CONTROL_PLANE_INSTANCE_TYPE="m6i.${CONTROL_PLANE_INSTANCE_SIZE}" + fi fi + elif [[ "${CONTROL_ARCH}" == "arm64" ]]; then + CONTROL_PLANE_INSTANCE_TYPE="m6g.${CONTROL_PLANE_INSTANCE_SIZE}" + else + echo "${CONTROL_ARCH} is not a valid control plane architecture..." + exit 1 fi - if [[ "${COMPUTE_NODE_TYPE}" == "" ]]; then - if [[ "${IS_M6A_REGION}" == "yes" ]]; then - COMPUTE_NODE_TYPE=$(eval_instance_capacity "m6a.xlarge" "m6i.xlarge") - else - COMPUTE_NODE_TYPE="m6i.xlarge" + if [[ "${COMPUTE_ARCH}" == "amd64" ]]; then + if [[ "${COMPUTE_NODE_TYPE}" == "" ]]; then + if [[ "${IS_M6A_REGION}" == "yes" ]]; then + COMPUTE_NODE_TYPE=$(eval_instance_capacity "m6a.xlarge" "m6i.xlarge") + else + COMPUTE_NODE_TYPE="m6i.xlarge" + fi fi + elif [[ "${COMPUTE_ARCH}" == "arm64" ]]; then + COMPUTE_NODE_TYPE="m6g.xlarge" + else + echo "${COMPUTE_ARCH} is not a valid compute plane architecture..." + exit 1 fi fi @@ -208,14 +220,14 @@ platform: userTags: expirationDate: ${expiration_date} controlPlane: - architecture: ${architecture} + architecture: ${CONTROL_ARCH} name: master replicas: ${master_replicas} platform: aws: type: ${CONTROL_PLANE_INSTANCE_TYPE} compute: -- architecture: ${architecture} +- architecture: ${COMPUTE_ARCH} name: worker replicas: ${worker_replicas} platform: @@ -339,7 +351,7 @@ if [[ -n "${AWS_EDGE_POOL_ENABLED-}" ]]; then patch_edge="${SHARED_DIR}/install-config-edge.yaml.patch" cat > "${patch_edge}" << EOF compute: -- architecture: ${architecture} +- architecture: ${COMPUTE_ARCH} name: edge platform: aws: diff --git a/ci-operator/step-registry/ipi/conf/aws/ipi-conf-aws-ref.yaml b/ci-operator/step-registry/ipi/conf/aws/ipi-conf-aws-ref.yaml index f10416b6a88fa..19584361cc71f 100644 --- a/ci-operator/step-registry/ipi/conf/aws/ipi-conf-aws-ref.yaml +++ b/ci-operator/step-registry/ipi/conf/aws/ipi-conf-aws-ref.yaml @@ -80,6 +80,14 @@ ref: * "" (default) - corresponds to "Optional" behaviour. * "Required" - sets IMDSv2 to Required for all machines in install-config, metadata service is not accessible without auth procedure. * "Optional" - auth procedure on metadata service is optional + - name: COMPUTE_ARCH + default: "amd64" + documentation: |- + Compute node architecture specification. Used for multiarch compute clusters. + - name: CONTROL_ARCH + default: "amd64" + documentation: |- + Control plane node architecture specification. Used for multiarch compute clusters. - name: PRESERVE_BOOTSTRAP_IGNITION default: "" documentation: |- diff --git a/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/day-0/OWNERS b/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/day-0/OWNERS new file mode 100644 index 0000000000000..4d59953a16274 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/day-0/OWNERS @@ -0,0 +1,5 @@ +approvers: +- deepsm007 +- multiarch-approvers +reviewers: +- multiarch-reviewers diff --git a/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/day-0/openshift-e2e-aws-heterogeneous-day-0-workflow.metadata.json b/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/day-0/openshift-e2e-aws-heterogeneous-day-0-workflow.metadata.json new file mode 100644 index 0000000000000..072e2b7e58bf7 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/day-0/openshift-e2e-aws-heterogeneous-day-0-workflow.metadata.json @@ -0,0 +1,12 @@ +{ + "path": "openshift/e2e/aws/heterogeneous/day-0/openshift-e2e-aws-heterogeneous-day-0-workflow.yaml", + "owners": { + "approvers": [ + "deepsm007", + "multiarch-approvers" + ], + "reviewers": [ + "multiarch-reviewers" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/day-0/openshift-e2e-aws-heterogeneous-day-0-workflow.yaml b/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/day-0/openshift-e2e-aws-heterogeneous-day-0-workflow.yaml new file mode 100644 index 0000000000000..04ca6eed489b3 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/aws/heterogeneous/day-0/openshift-e2e-aws-heterogeneous-day-0-workflow.yaml @@ -0,0 +1,30 @@ +workflow: + as: openshift-e2e-aws-heterogeneous-day-0 + steps: + allow_best_effort_post_steps: true + pre: + - chain: ipi-aws-pre + test: + - ref: openshift-e2e-test + post: + - chain: gather-core-dump + - chain: ipi-aws-post + env: + TEST_SKIPS: >- + The HAProxy router should\| deploymentconfigs\| Users should be able to create and run a job in a user project\| + Managed cluster should expose cluster services outside the cluster\| Managed cluster should should expose cluster services outside the cluster\| FIPS TestFIPS\| + TestPodDefaultCapabilities\| Multi-stage image builds should succeed\| Optimized image builds should succeed\| + build can reference a cluster service with a build being created from new-build should be able to run a build that references a cluster service\| + custom build with buildah being created from new-build should complete build with custom builder image\| + oc new-app should succeed with a --name of 58 characters\| oc can run inside of a busybox container\| + oc debug deployment configs from a build\| oc rsh specific flags should work well when access to a remote shell\| + prune builds based on settings in the buildconfig should prune completed builds based on the successfulBuildsHistoryLimit setting\| + s2i build with a\| verify /run filesystem contents\| Pods cannot access the /config/master API endpoint\| + Image extract should extract content from an image\| Image info should display information about images\| + Image layer subresource should return layers from tagged images\| oc tag should change image reference for internal images\| + when installed on the cluster should\| OpenShift alerting rules\| Image append should create images by appending them\| + egressFirewall should have no impact outside its namespace\| should ensure ipv4 egressrouter cni resources are created\| + pod should start with all sysctl on whitelist\| pod sysctls should not affect node\| + when FIPS is disabled the HAProxy router should serve routes when configured with a 1024-bit RSA key + documentation: |- + The Openshift E2E HETEROGENEOUS AWS workflow executes the common end-to-end test suite on AWS with a heterogeneous cluster.