From f594928971c7d3b86e3513d2fcf4b53c850d010a Mon Sep 17 00:00:00 2001 From: Eric Fried Date: Fri, 3 May 2024 18:13:47 -0500 Subject: [PATCH 1/2] Enable spot instances for AWS masters Add a new variable for the AWS IPI flows, `$SPOT_MASTERS`. When using CAPI installs (installer will generate the `cluster-api` directory) *or* OCP versions containing https://github.com/openshift/installer/pull/8349 this can be set to `'true'` to inject `spotMarketOptions` into master machine manifests. The existing `$SPOT_INSTANCES` variable is unchanged: as before, it only results in *worker* nodes using spot instances. (We may at some point wish to rename this to `$SPOT_WORKERS` for clarity.) NOTE: Spot instances are unreliable. Using them may cause additional flakes in your tests. --- .../install/ipi-install-install-commands.sh | 111 ++++++++++++++---- .../install/ipi-install-install-ref.yaml | 5 +- 2 files changed, 90 insertions(+), 26 deletions(-) diff --git a/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh b/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh index a47b3a2f916ef..91e2d1f9d6f2b 100755 --- a/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh +++ b/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh @@ -425,14 +425,18 @@ EOF /tmp/fcct --pretty --strict -d "${config_dir}" "${config_dir}/fcct.yml" > "${dir}/bootstrap.ign" } +function get_yq() { + if [ ! -f /tmp/yq ]; then + curl -L "https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_$( get_arch )" \ + -o /tmp/yq && chmod +x /tmp/yq || exit 1 + fi +} + # inject_boot_diagnostics is an azure specific function for enabling boot diagnostics on Azure workers. function inject_boot_diagnostics() { local dir=${1} - if [ ! -f /tmp/yq ]; then - curl -L "https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_$( get_arch )" \ - -o /tmp/yq && chmod +x /tmp/yq - fi + get_yq PATCH="${SHARED_DIR}/machinesets-boot-diagnostics.yaml.patch" cat > "${PATCH}" << EOF @@ -451,31 +455,85 @@ EOF done } -# inject_spot_instance_config is an AWS specific option that enables the use of AWS spot instances for worker nodes +# inject_spot_instance_config is an AWS specific option that enables the +# use of AWS spot instances. +# PARAMS: +# $1: Path to base output directory of `openshift-install create manifests` +# $2: Either "workers" or "masters" to enable spot instances on the +# compute or control machines, respectively. function inject_spot_instance_config() { local dir=${1} + local mtype=${2} + + get_yq + + # Find manifest files + local manifests= + case "${mtype}" in + masters) + manifests="${dir}/openshift/99_openshift-machine-api_master-control-plane-machine-set.yaml \ + ${dir}/openshift/99_openshift-cluster-api_*-machines-*.yaml" + # Spot masters works for + # - CAPA, always -- discover by existence of the cluster-api directory + # - Terraform, only for newer installer binaries containing https://github.com/openshift/installer/pull/8349 + if [[ -d ${dir}/cluster-api/machines ]]; then + echo "Spot masters supported via CAPA" + manifests="${dir}/cluster-api/machines/10_inframachine_*.yaml $manifests" + elif openshift-install list-hidden-features 2>/dev/null | grep -q terraform-spot-masters; then + echo "Spot masters supported via terraform" + else + echo "Spot masters are not supported in this configuration!" + exit 1 + fi + ;; + workers) + manifests="${dir}/openshift/99_openshift-cluster-api_*-machineset-*.yaml" + ;; + *) + echo "ERROR: Invalid machine type '$mtype' passed to inject_spot_instance_config; expected 'masters' or 'workers'" + exit 1 + ;; + esac - if [ ! -f /tmp/yq ]; then - curl -L "https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_$( get_arch )" \ - -o /tmp/yq && chmod +x /tmp/yq - fi - - PATCH="${SHARED_DIR}/machinesets-spot-instances.yaml.patch" - cat > "${PATCH}" << EOF -spec: - template: - spec: - providerSpec: - value: - spotMarketOptions: {} -EOF - - for MACHINESET in $dir/openshift/99_openshift-cluster-api_worker-machineset-*.yaml; do - /tmp/yq m -x -i "${MACHINESET}" "${PATCH}" - echo "Patched spotMarketOptions into ${MACHINESET}" + # Inject spotMarketOptions into the appropriate manifests + local prefix= + local found=false + # Don't rely on file names; iterate through all the manifests and match + # by kind. + for manifest in $manifests; do + kind=$(/tmp/yq r "${manifest}" kind) + case "${kind}" in + MachineSet) # Workers, both tf and CAPA, run through MachineSet today. + [[ "${mtype}" == "workers" ]] || continue + prefix='spec.template.spec.providerSpec.value' + ;; + AWSMachine) # CAPA masters + [[ "${mtype}" == "masters" ]] || continue + prefix='spec' + ;; + Machine) # tf masters during install + [[ "${mtype}" == "masters" ]] || continue + prefix='spec.providerSpec.value' + ;; + ControlPlaneMachineSet) # masters reconciled after install + [[ "${mtype}" == "masters" ]] || continue + prefix='spec.template.machines_v1beta1_machine_openshift_io.spec.providerSpec.value' + ;; + *) + continue + ;; + esac + found=true + echo "Using spot instances for ${kind} in ${manifest}" + /tmp/yq w -i --tag '!!str' "${manifest}" "${prefix}.spotMarketOptions.maxPrice" '' done - echo "Enabled AWS Spot instances for worker nodes" + if $found; then + echo "Enabled AWS Spot instances for ${mtype}" + else + echo "ERROR: Spot instances were requested for ${mtype}, but no such manifests were found!" + exit 1 + fi } # enable_efa_pg_instance_config is an AWS specific option that enables one worker machineset in a placement group and with EFA Network Interface Type, other worker machinesets will be ENA Network Interface Type by default..... @@ -628,7 +686,10 @@ case "${CLUSTER_TYPE}" in azure4|azure-arm64) inject_boot_diagnostics ${dir} ;; aws|aws-arm64|aws-usgov) if [[ "${SPOT_INSTANCES:-}" == 'true' ]]; then - inject_spot_instance_config ${dir} + inject_spot_instance_config "${dir}" "workers" + fi + if [[ "${SPOT_MASTERS:-}" == 'true' ]]; then + inject_spot_instance_config "${dir}" "masters" fi if [[ "${ENABLE_AWS_EFA_PG_INSTANCE:-}" == 'true' ]]; then enable_efa_pg_instance_config ${dir} diff --git a/ci-operator/step-registry/ipi/install/install/ipi-install-install-ref.yaml b/ci-operator/step-registry/ipi/install/install/ipi-install-install-ref.yaml index c33068f79519d..7d474bfc5c7ca 100644 --- a/ci-operator/step-registry/ipi/install/install/ipi-install-install-ref.yaml +++ b/ci-operator/step-registry/ipi/install/install/ipi-install-install-ref.yaml @@ -45,7 +45,10 @@ ref: documentation: "Specifies the logging level for terraform provider libraries." - name: SPOT_INSTANCES default: "false" - documentation: "Use AWS Spot Instances for worker nodes. Set to 'true' to opt into spot instances. Explicitly set to 'false' to opt out. Leave unset for the default, which may change." + documentation: "Use AWS Spot Instances for *worker* nodes. Set to 'true' to opt into spot instances. Explicitly set to 'false' to opt out. Leave unset for the default, which may change." + - name: SPOT_MASTERS + default: "false" + documentation: "Use AWS Spot Instances for *master* nodes. Set to 'true' to opt into spot instances. Explicitly set to 'false' to opt out. Leave unset for the default, which may change. Note that spot masters are only supported when installing with a) CAPI; or b) newer installer versions (see https://github.com/openshift/installer/pull/8349). A preflight check will fail if this variable is set to 'true' for an unsupported configuration." - name: OPENSHIFT_INSTALL_AWS_PUBLIC_ONLY default: "" documentation: "Whether to use only public subnets for AWS. Implies no NAT Gateways. Requires a VPC to be configured prior to install." From 1bac135484ebd986073a28550e91a035259d2cbd Mon Sep 17 00:00:00 2001 From: Eric Fried Date: Mon, 6 May 2024 09:49:32 -0500 Subject: [PATCH 2/2] installer / master, 4.17, 4.16: AWS spot instances Convert all AWS IPI-based presubmits in the openshift-installer legacy (terraform) and altinfra (CAPI) test suites to use spot instances for branches: - master - release-4.18 - release-4.17 - release-4.16 --- .../installer/openshift-installer-master.yaml | 25 +++++++++++++++++++ .../openshift-installer-master__altinfra.yaml | 24 ++++++++++++++++++ .../openshift-installer-release-4.16.yaml | 25 +++++++++++++++++++ ...hift-installer-release-4.16__altinfra.yaml | 24 ++++++++++++++++++ .../openshift-installer-release-4.17.yaml | 25 +++++++++++++++++++ ...hift-installer-release-4.17__altinfra.yaml | 24 ++++++++++++++++++ .../openshift-installer-release-4.18.yaml | 25 +++++++++++++++++++ ...hift-installer-release-4.18__altinfra.yaml | 24 ++++++++++++++++++ 8 files changed, 196 insertions(+) diff --git a/ci-operator/config/openshift/installer/openshift-installer-master.yaml b/ci-operator/config/openshift/installer/openshift-installer-master.yaml index 4f11a6ff44152..9ca9afd51d4ee 100644 --- a/ci-operator/config/openshift/installer/openshift-installer-master.yaml +++ b/ci-operator/config/openshift/installer/openshift-installer-master.yaml @@ -357,6 +357,8 @@ tests: steps: cluster_profile: aws env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" USER_TAGS: | keyA valueA keyB valueB @@ -368,6 +370,9 @@ tests: optional: true steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-proxy timeout: 6h0m0s - always_run: false @@ -382,6 +387,9 @@ tests: run_if_changed: aws steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" post: - chain: ipi-aws-post - ref: ipi-deprovision-aws-custom-security-groups @@ -425,6 +433,8 @@ tests: cluster_profile: aws env: AWS_METADATA_SERVICE_AUTH: Required + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws timeout: 6h0m0s - as: e2e-gcp-ovn @@ -471,6 +481,9 @@ tests: optional: true steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-upgrade-aws-latestinstaller timeout: 6h0m0s - as: e2e-aws-ovn-upi @@ -512,6 +525,8 @@ tests: cluster_profile: aws env: PLATFORM_VERSION: "8.4" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-workers-rhel timeout: 6h0m0s - always_run: false @@ -731,6 +746,9 @@ tests: run_if_changed: aws steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-single-node timeout: 6h0m0s - as: e2e-metal-single-node-live-iso @@ -747,6 +765,8 @@ tests: cluster_profile: aws env: FIPS_ENABLED: "true" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" TEST_SUITE: openshift/conformance/parallel pre: - chain: ipi-aws-pre @@ -858,6 +878,9 @@ tests: run_if_changed: /aws/subnet.go steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-publicsubnets timeout: 6h0m0s - always_run: false @@ -873,6 +896,8 @@ tests: cluster_profile: aws env: CLUSTER_NETWORK_MTU: "1200" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-network-mtu timeout: 6h0m0s - always_run: false diff --git a/ci-operator/config/openshift/installer/openshift-installer-master__altinfra.yaml b/ci-operator/config/openshift/installer/openshift-installer-master__altinfra.yaml index 6af67eb3307e3..c9a346f9b377a 100644 --- a/ci-operator/config/openshift/installer/openshift-installer-master__altinfra.yaml +++ b/ci-operator/config/openshift/installer/openshift-installer-master__altinfra.yaml @@ -98,6 +98,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-custom-security-groups timeout: 6h0m0s - always_run: false @@ -110,6 +112,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" USER_TAGS: | keyA valueA keyB valueB @@ -126,6 +130,8 @@ tests: FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade FIPS_ENABLED: "true" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" TEST_SUITE: openshift/conformance/parallel pre: - chain: ipi-aws-pre @@ -144,6 +150,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws timeout: 6h0m0s - always_run: false @@ -154,6 +162,8 @@ tests: env: FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-proxy timeout: 6h0m0s - always_run: false @@ -164,6 +174,8 @@ tests: env: FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" USER_TAGS: | keyA valueA keyB valueB @@ -182,6 +194,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-sharednetwork timeout: 6h0m0s - always_run: false @@ -193,6 +207,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-single-node timeout: 6h0m0s - always_run: false @@ -205,6 +221,8 @@ tests: EDGE_ZONE_TYPES: local-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones timeout: 6h0m0s - always_run: false @@ -217,6 +235,8 @@ tests: EDGE_ZONE_TYPES: wavelength-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones timeout: 6h0m0s - always_run: false @@ -229,6 +249,8 @@ tests: EDGE_ZONE_TYPES: local-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones-byo-vpc timeout: 6h0m0s - always_run: false @@ -241,6 +263,8 @@ tests: EDGE_ZONE_TYPES: wavelength-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones-byo-vpc timeout: 6h0m0s - always_run: false diff --git a/ci-operator/config/openshift/installer/openshift-installer-release-4.16.yaml b/ci-operator/config/openshift/installer/openshift-installer-release-4.16.yaml index a7517fa2cfedf..2f8fad1daab6b 100644 --- a/ci-operator/config/openshift/installer/openshift-installer-release-4.16.yaml +++ b/ci-operator/config/openshift/installer/openshift-installer-release-4.16.yaml @@ -307,6 +307,8 @@ tests: steps: cluster_profile: aws env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" USER_TAGS: | keyA valueA keyB valueB @@ -318,6 +320,9 @@ tests: optional: true steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-proxy timeout: 6h0m0s - always_run: false @@ -332,6 +337,9 @@ tests: run_if_changed: aws steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" post: - chain: ipi-aws-post - ref: ipi-deprovision-aws-custom-security-groups @@ -375,6 +383,8 @@ tests: cluster_profile: aws env: AWS_METADATA_SERVICE_AUTH: Required + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws timeout: 6h0m0s - as: e2e-gcp-ovn @@ -421,6 +431,9 @@ tests: optional: true steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-upgrade-aws-latestinstaller timeout: 6h0m0s - as: e2e-aws-ovn-upi @@ -462,6 +475,8 @@ tests: cluster_profile: aws env: PLATFORM_VERSION: "8.4" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-workers-rhel timeout: 6h0m0s - always_run: false @@ -681,6 +696,9 @@ tests: run_if_changed: aws steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-single-node timeout: 6h0m0s - as: e2e-metal-single-node-live-iso @@ -697,6 +715,8 @@ tests: cluster_profile: aws env: FIPS_ENABLED: "true" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" TEST_SUITE: openshift/conformance/parallel pre: - chain: ipi-aws-pre @@ -808,6 +828,9 @@ tests: run_if_changed: /aws/subnet.go steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-publicsubnets timeout: 6h0m0s - always_run: false @@ -823,6 +846,8 @@ tests: cluster_profile: aws env: CLUSTER_NETWORK_MTU: "1200" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-network-mtu timeout: 6h0m0s - always_run: false diff --git a/ci-operator/config/openshift/installer/openshift-installer-release-4.16__altinfra.yaml b/ci-operator/config/openshift/installer/openshift-installer-release-4.16__altinfra.yaml index 947cf542512a8..b2a320fc190f9 100644 --- a/ci-operator/config/openshift/installer/openshift-installer-release-4.16__altinfra.yaml +++ b/ci-operator/config/openshift/installer/openshift-installer-release-4.16__altinfra.yaml @@ -98,6 +98,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-custom-security-groups timeout: 6h0m0s - always_run: false @@ -110,6 +112,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" USER_TAGS: | keyA valueA keyB valueB @@ -126,6 +130,8 @@ tests: FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade FIPS_ENABLED: "true" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" TEST_SUITE: openshift/conformance/parallel pre: - chain: ipi-aws-pre @@ -144,6 +150,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws timeout: 6h0m0s - always_run: false @@ -154,6 +162,8 @@ tests: env: FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-proxy timeout: 6h0m0s - always_run: false @@ -164,6 +174,8 @@ tests: env: FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" USER_TAGS: | keyA valueA keyB valueB @@ -182,6 +194,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-sharednetwork timeout: 6h0m0s - always_run: false @@ -193,6 +207,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-single-node timeout: 6h0m0s - always_run: false @@ -205,6 +221,8 @@ tests: EDGE_ZONE_TYPES: local-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones timeout: 6h0m0s - always_run: false @@ -217,6 +235,8 @@ tests: EDGE_ZONE_TYPES: wavelength-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones timeout: 6h0m0s - always_run: false @@ -229,6 +249,8 @@ tests: EDGE_ZONE_TYPES: local-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones-byo-vpc timeout: 6h0m0s - always_run: false @@ -241,6 +263,8 @@ tests: EDGE_ZONE_TYPES: wavelength-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones-byo-vpc timeout: 6h0m0s - always_run: false diff --git a/ci-operator/config/openshift/installer/openshift-installer-release-4.17.yaml b/ci-operator/config/openshift/installer/openshift-installer-release-4.17.yaml index 65f20f4485d3b..16afad817dcb4 100644 --- a/ci-operator/config/openshift/installer/openshift-installer-release-4.17.yaml +++ b/ci-operator/config/openshift/installer/openshift-installer-release-4.17.yaml @@ -358,6 +358,8 @@ tests: steps: cluster_profile: aws env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" USER_TAGS: | keyA valueA keyB valueB @@ -369,6 +371,9 @@ tests: optional: true steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-proxy timeout: 6h0m0s - always_run: false @@ -383,6 +388,9 @@ tests: run_if_changed: aws steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" post: - chain: ipi-aws-post - ref: ipi-deprovision-aws-custom-security-groups @@ -426,6 +434,8 @@ tests: cluster_profile: aws env: AWS_METADATA_SERVICE_AUTH: Required + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws timeout: 6h0m0s - as: e2e-gcp-ovn @@ -472,6 +482,9 @@ tests: optional: true steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-upgrade-aws-latestinstaller timeout: 6h0m0s - as: e2e-aws-ovn-upi @@ -513,6 +526,8 @@ tests: cluster_profile: aws env: PLATFORM_VERSION: "8.4" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-workers-rhel timeout: 6h0m0s - always_run: false @@ -732,6 +747,9 @@ tests: run_if_changed: aws steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-single-node timeout: 6h0m0s - as: e2e-metal-single-node-live-iso @@ -748,6 +766,8 @@ tests: cluster_profile: aws env: FIPS_ENABLED: "true" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" TEST_SUITE: openshift/conformance/parallel pre: - chain: ipi-aws-pre @@ -859,6 +879,9 @@ tests: run_if_changed: /aws/subnet.go steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-publicsubnets timeout: 6h0m0s - always_run: false @@ -874,6 +897,8 @@ tests: cluster_profile: aws env: CLUSTER_NETWORK_MTU: "1200" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-network-mtu timeout: 6h0m0s - always_run: false diff --git a/ci-operator/config/openshift/installer/openshift-installer-release-4.17__altinfra.yaml b/ci-operator/config/openshift/installer/openshift-installer-release-4.17__altinfra.yaml index 27c4bd38ee890..3e8a1d750b6fb 100644 --- a/ci-operator/config/openshift/installer/openshift-installer-release-4.17__altinfra.yaml +++ b/ci-operator/config/openshift/installer/openshift-installer-release-4.17__altinfra.yaml @@ -99,6 +99,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-custom-security-groups timeout: 6h0m0s - always_run: false @@ -111,6 +113,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" USER_TAGS: | keyA valueA keyB valueB @@ -127,6 +131,8 @@ tests: FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade FIPS_ENABLED: "true" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" TEST_SUITE: openshift/conformance/parallel pre: - chain: ipi-aws-pre @@ -145,6 +151,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws timeout: 6h0m0s - always_run: false @@ -155,6 +163,8 @@ tests: env: FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-proxy timeout: 6h0m0s - always_run: false @@ -165,6 +175,8 @@ tests: env: FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" USER_TAGS: | keyA valueA keyB valueB @@ -183,6 +195,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-sharednetwork timeout: 6h0m0s - always_run: false @@ -194,6 +208,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-single-node timeout: 6h0m0s - always_run: false @@ -206,6 +222,8 @@ tests: EDGE_ZONE_TYPES: local-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones timeout: 6h0m0s - always_run: false @@ -218,6 +236,8 @@ tests: EDGE_ZONE_TYPES: wavelength-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones timeout: 6h0m0s - always_run: false @@ -230,6 +250,8 @@ tests: EDGE_ZONE_TYPES: local-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones-byo-vpc timeout: 6h0m0s - always_run: false @@ -242,6 +264,8 @@ tests: EDGE_ZONE_TYPES: wavelength-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones-byo-vpc timeout: 6h0m0s - always_run: false diff --git a/ci-operator/config/openshift/installer/openshift-installer-release-4.18.yaml b/ci-operator/config/openshift/installer/openshift-installer-release-4.18.yaml index 362683c47451c..f6021e52ec4fd 100644 --- a/ci-operator/config/openshift/installer/openshift-installer-release-4.18.yaml +++ b/ci-operator/config/openshift/installer/openshift-installer-release-4.18.yaml @@ -357,6 +357,8 @@ tests: steps: cluster_profile: aws env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" USER_TAGS: | keyA valueA keyB valueB @@ -368,6 +370,9 @@ tests: optional: true steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-proxy timeout: 6h0m0s - always_run: false @@ -382,6 +387,9 @@ tests: run_if_changed: aws steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" post: - chain: ipi-aws-post - ref: ipi-deprovision-aws-custom-security-groups @@ -425,6 +433,8 @@ tests: cluster_profile: aws env: AWS_METADATA_SERVICE_AUTH: Required + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws timeout: 6h0m0s - as: e2e-gcp-ovn @@ -471,6 +481,9 @@ tests: optional: true steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-upgrade-aws-latestinstaller timeout: 6h0m0s - as: e2e-aws-ovn-upi @@ -512,6 +525,8 @@ tests: cluster_profile: aws env: PLATFORM_VERSION: "8.4" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-workers-rhel timeout: 6h0m0s - always_run: false @@ -731,6 +746,9 @@ tests: run_if_changed: aws steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-single-node timeout: 6h0m0s - as: e2e-metal-single-node-live-iso @@ -747,6 +765,8 @@ tests: cluster_profile: aws env: FIPS_ENABLED: "true" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" TEST_SUITE: openshift/conformance/parallel pre: - chain: ipi-aws-pre @@ -858,6 +878,9 @@ tests: run_if_changed: /aws/subnet.go steps: cluster_profile: aws + env: + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-publicsubnets timeout: 6h0m0s - always_run: false @@ -873,6 +896,8 @@ tests: cluster_profile: aws env: CLUSTER_NETWORK_MTU: "1200" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-network-mtu timeout: 6h0m0s - always_run: false diff --git a/ci-operator/config/openshift/installer/openshift-installer-release-4.18__altinfra.yaml b/ci-operator/config/openshift/installer/openshift-installer-release-4.18__altinfra.yaml index 3fc1bcc5f1ab3..88101622d2d4d 100644 --- a/ci-operator/config/openshift/installer/openshift-installer-release-4.18__altinfra.yaml +++ b/ci-operator/config/openshift/installer/openshift-installer-release-4.18__altinfra.yaml @@ -98,6 +98,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-custom-security-groups timeout: 6h0m0s - always_run: false @@ -110,6 +112,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" USER_TAGS: | keyA valueA keyB valueB @@ -126,6 +130,8 @@ tests: FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade FIPS_ENABLED: "true" + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" TEST_SUITE: openshift/conformance/parallel pre: - chain: ipi-aws-pre @@ -144,6 +150,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws timeout: 6h0m0s - always_run: false @@ -154,6 +162,8 @@ tests: env: FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-proxy timeout: 6h0m0s - always_run: false @@ -164,6 +174,8 @@ tests: env: FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" USER_TAGS: | keyA valueA keyB valueB @@ -182,6 +194,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-sharednetwork timeout: 6h0m0s - always_run: false @@ -193,6 +207,8 @@ tests: AWS_PUBLIC_IPV4_POOL_ID: none FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-single-node timeout: 6h0m0s - always_run: false @@ -205,6 +221,8 @@ tests: EDGE_ZONE_TYPES: local-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones timeout: 6h0m0s - always_run: false @@ -217,6 +235,8 @@ tests: EDGE_ZONE_TYPES: wavelength-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones timeout: 6h0m0s - always_run: false @@ -229,6 +249,8 @@ tests: EDGE_ZONE_TYPES: local-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones-byo-vpc timeout: 6h0m0s - always_run: false @@ -241,6 +263,8 @@ tests: EDGE_ZONE_TYPES: wavelength-zone FEATURE_GATES: '["ClusterAPIInstall=true"]' FEATURE_SET: CustomNoUpgrade + SPOT_INSTANCES: "true" + SPOT_MASTERS: "true" workflow: openshift-e2e-aws-edge-zones-byo-vpc timeout: 6h0m0s - always_run: false