diff --git a/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh b/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh index a47b3a2f916ef..91e2d1f9d6f2b 100755 --- a/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh +++ b/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh @@ -425,14 +425,18 @@ EOF /tmp/fcct --pretty --strict -d "${config_dir}" "${config_dir}/fcct.yml" > "${dir}/bootstrap.ign" } +function get_yq() { + if [ ! -f /tmp/yq ]; then + curl -L "https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_$( get_arch )" \ + -o /tmp/yq && chmod +x /tmp/yq || exit 1 + fi +} + # inject_boot_diagnostics is an azure specific function for enabling boot diagnostics on Azure workers. function inject_boot_diagnostics() { local dir=${1} - if [ ! -f /tmp/yq ]; then - curl -L "https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_$( get_arch )" \ - -o /tmp/yq && chmod +x /tmp/yq - fi + get_yq PATCH="${SHARED_DIR}/machinesets-boot-diagnostics.yaml.patch" cat > "${PATCH}" << EOF @@ -451,31 +455,85 @@ EOF done } -# inject_spot_instance_config is an AWS specific option that enables the use of AWS spot instances for worker nodes +# inject_spot_instance_config is an AWS specific option that enables the +# use of AWS spot instances. +# PARAMS: +# $1: Path to base output directory of `openshift-install create manifests` +# $2: Either "workers" or "masters" to enable spot instances on the +# compute or control machines, respectively. function inject_spot_instance_config() { local dir=${1} + local mtype=${2} + + get_yq + + # Find manifest files + local manifests= + case "${mtype}" in + masters) + manifests="${dir}/openshift/99_openshift-machine-api_master-control-plane-machine-set.yaml \ + ${dir}/openshift/99_openshift-cluster-api_*-machines-*.yaml" + # Spot masters works for + # - CAPA, always -- discover by existence of the cluster-api directory + # - Terraform, only for newer installer binaries containing https://github.com/openshift/installer/pull/8349 + if [[ -d ${dir}/cluster-api/machines ]]; then + echo "Spot masters supported via CAPA" + manifests="${dir}/cluster-api/machines/10_inframachine_*.yaml $manifests" + elif openshift-install list-hidden-features 2>/dev/null | grep -q terraform-spot-masters; then + echo "Spot masters supported via terraform" + else + echo "Spot masters are not supported in this configuration!" + exit 1 + fi + ;; + workers) + manifests="${dir}/openshift/99_openshift-cluster-api_*-machineset-*.yaml" + ;; + *) + echo "ERROR: Invalid machine type '$mtype' passed to inject_spot_instance_config; expected 'masters' or 'workers'" + exit 1 + ;; + esac - if [ ! -f /tmp/yq ]; then - curl -L "https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_$( get_arch )" \ - -o /tmp/yq && chmod +x /tmp/yq - fi - - PATCH="${SHARED_DIR}/machinesets-spot-instances.yaml.patch" - cat > "${PATCH}" << EOF -spec: - template: - spec: - providerSpec: - value: - spotMarketOptions: {} -EOF - - for MACHINESET in $dir/openshift/99_openshift-cluster-api_worker-machineset-*.yaml; do - /tmp/yq m -x -i "${MACHINESET}" "${PATCH}" - echo "Patched spotMarketOptions into ${MACHINESET}" + # Inject spotMarketOptions into the appropriate manifests + local prefix= + local found=false + # Don't rely on file names; iterate through all the manifests and match + # by kind. + for manifest in $manifests; do + kind=$(/tmp/yq r "${manifest}" kind) + case "${kind}" in + MachineSet) # Workers, both tf and CAPA, run through MachineSet today. + [[ "${mtype}" == "workers" ]] || continue + prefix='spec.template.spec.providerSpec.value' + ;; + AWSMachine) # CAPA masters + [[ "${mtype}" == "masters" ]] || continue + prefix='spec' + ;; + Machine) # tf masters during install + [[ "${mtype}" == "masters" ]] || continue + prefix='spec.providerSpec.value' + ;; + ControlPlaneMachineSet) # masters reconciled after install + [[ "${mtype}" == "masters" ]] || continue + prefix='spec.template.machines_v1beta1_machine_openshift_io.spec.providerSpec.value' + ;; + *) + continue + ;; + esac + found=true + echo "Using spot instances for ${kind} in ${manifest}" + /tmp/yq w -i --tag '!!str' "${manifest}" "${prefix}.spotMarketOptions.maxPrice" '' done - echo "Enabled AWS Spot instances for worker nodes" + if $found; then + echo "Enabled AWS Spot instances for ${mtype}" + else + echo "ERROR: Spot instances were requested for ${mtype}, but no such manifests were found!" + exit 1 + fi } # enable_efa_pg_instance_config is an AWS specific option that enables one worker machineset in a placement group and with EFA Network Interface Type, other worker machinesets will be ENA Network Interface Type by default..... @@ -628,7 +686,10 @@ case "${CLUSTER_TYPE}" in azure4|azure-arm64) inject_boot_diagnostics ${dir} ;; aws|aws-arm64|aws-usgov) if [[ "${SPOT_INSTANCES:-}" == 'true' ]]; then - inject_spot_instance_config ${dir} + inject_spot_instance_config "${dir}" "workers" + fi + if [[ "${SPOT_MASTERS:-}" == 'true' ]]; then + inject_spot_instance_config "${dir}" "masters" fi if [[ "${ENABLE_AWS_EFA_PG_INSTANCE:-}" == 'true' ]]; then enable_efa_pg_instance_config ${dir} diff --git a/ci-operator/step-registry/ipi/install/install/ipi-install-install-ref.yaml b/ci-operator/step-registry/ipi/install/install/ipi-install-install-ref.yaml index c33068f79519d..7d474bfc5c7ca 100644 --- a/ci-operator/step-registry/ipi/install/install/ipi-install-install-ref.yaml +++ b/ci-operator/step-registry/ipi/install/install/ipi-install-install-ref.yaml @@ -45,7 +45,10 @@ ref: documentation: "Specifies the logging level for terraform provider libraries." - name: SPOT_INSTANCES default: "false" - documentation: "Use AWS Spot Instances for worker nodes. Set to 'true' to opt into spot instances. Explicitly set to 'false' to opt out. Leave unset for the default, which may change." + documentation: "Use AWS Spot Instances for *worker* nodes. Set to 'true' to opt into spot instances. Explicitly set to 'false' to opt out. Leave unset for the default, which may change." + - name: SPOT_MASTERS + default: "false" + documentation: "Use AWS Spot Instances for *master* nodes. Set to 'true' to opt into spot instances. Explicitly set to 'false' to opt out. Leave unset for the default, which may change. Note that spot masters are only supported when installing with a) CAPI; or b) newer installer versions (see https://github.com/openshift/installer/pull/8349). A preflight check will fail if this variable is set to 'true' for an unsupported configuration." - name: OPENSHIFT_INSTALL_AWS_PUBLIC_ONLY default: "" documentation: "Whether to use only public subnets for AWS. Implies no NAT Gateways. Requires a VPC to be configured prior to install."