Enable spot instances for AWS masters

ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh

@@ -425,14 +425,18 @@ EOF
   /tmp/fcct --pretty --strict -d "${config_dir}" "${config_dir}/fcct.yml" > "${dir}/bootstrap.ign"
 }
 
+function get_yq() {
+  if [ ! -f /tmp/yq ]; then
+    curl -L "https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_$( get_arch )" \
+    -o /tmp/yq && chmod +x /tmp/yq || exit 1
+  fi
+}
+
 # inject_boot_diagnostics is an azure specific function for enabling boot diagnostics on Azure workers.
 function inject_boot_diagnostics() {
   local dir=${1}
 
-  if [ ! -f /tmp/yq ]; then
-    curl -L "https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_$( get_arch )" \
-    -o /tmp/yq && chmod +x /tmp/yq
-  fi
+  get_yq
 
   PATCH="${SHARED_DIR}/machinesets-boot-diagnostics.yaml.patch"
   cat > "${PATCH}" << EOF
@@ -451,31 +455,85 @@ EOF
   done
 }
 
-# inject_spot_instance_config is an AWS specific option that enables the use of AWS spot instances for worker nodes
+# inject_spot_instance_config is an AWS specific option that enables the
+# use of AWS spot instances.
+# PARAMS:
+# $1: Path to base output directory of `openshift-install create manifests`
+# $2: Either "workers" or "masters" to enable spot instances on the
+#     compute or control machines, respectively.
 function inject_spot_instance_config() {
   local dir=${1}
+  local mtype=${2}
+
+  get_yq
+
+  # Find manifest files
+  local manifests=
+  case "${mtype}" in
+    masters)
+      manifests="${dir}/openshift/99_openshift-machine-api_master-control-plane-machine-set.yaml \
+        ${dir}/openshift/99_openshift-cluster-api_*-machines-*.yaml"
+      # Spot masters works for
+      # - CAPA, always -- discover by existence of the cluster-api directory
+      # - Terraform, only for newer installer binaries containing https://github.com/openshift/installer/pull/8349
+      if [[ -d ${dir}/cluster-api/machines ]]; then
+        echo "Spot masters supported via CAPA"
+        manifests="${dir}/cluster-api/machines/10_inframachine_*.yaml $manifests"
+      elif openshift-install list-hidden-features 2>/dev/null | grep -q terraform-spot-masters; then
+        echo "Spot masters supported via terraform"
+      else
+        echo "Spot masters are not supported in this configuration!"
+        exit 1
+      fi
+      ;;
+    workers)
+      manifests="${dir}/openshift/99_openshift-cluster-api_*-machineset-*.yaml"
+      ;;
+    *)
+      echo "ERROR: Invalid machine type '$mtype' passed to inject_spot_instance_config; expected 'masters' or 'workers'"
+      exit 1
+      ;;
+  esac
 
-  if [ ! -f /tmp/yq ]; then
-    curl -L "https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_$( get_arch )" \
-    -o /tmp/yq && chmod +x /tmp/yq
-  fi
-
-  PATCH="${SHARED_DIR}/machinesets-spot-instances.yaml.patch"
-  cat > "${PATCH}" << EOF
-spec:
-  template:
-    spec:
-      providerSpec:
-        value:
-          spotMarketOptions: {}
-EOF
-
-  for MACHINESET in $dir/openshift/99_openshift-cluster-api_worker-machineset-*.yaml; do
-    /tmp/yq m -x -i "${MACHINESET}" "${PATCH}"
-    echo "Patched spotMarketOptions into ${MACHINESET}"
+  # Inject spotMarketOptions into the appropriate manifests
+  local prefix=
+  local found=false
+  # Don't rely on file names; iterate through all the manifests and match
+  # by kind.
+  for manifest in $manifests; do
+    kind=$(/tmp/yq r "${manifest}" kind)
+    case "${kind}" in
+      MachineSet)  # Workers, both tf and CAPA, run through MachineSet today.
+          [[ "${mtype}" == "workers" ]] || continue
+          prefix='spec.template.spec.providerSpec.value'
+          ;;
+      AWSMachine)  # CAPA masters
+          [[ "${mtype}" == "masters" ]] || continue
+          prefix='spec'
+          ;;
+      Machine)  # tf masters during install
+          [[ "${mtype}" == "masters" ]] || continue
+          prefix='spec.providerSpec.value'
+          ;;
+      ControlPlaneMachineSet)  # masters reconciled after install
+          [[ "${mtype}" == "masters" ]] || continue
+          prefix='spec.template.machines_v1beta1_machine_openshift_io.spec.providerSpec.value'
+          ;;
+      *)
+          continue
+          ;;
+    esac
+    found=true
+    echo "Using spot instances for ${kind} in ${manifest}"
+    /tmp/yq w -i --tag '!!str' "${manifest}" "${prefix}.spotMarketOptions.maxPrice" ''
   done
 
-  echo "Enabled AWS Spot instances for worker nodes"
+  if $found; then
+    echo "Enabled AWS Spot instances for ${mtype}"
+  else
+    echo "ERROR: Spot instances were requested for ${mtype}, but no such manifests were found!"
+    exit 1
+  fi
 }
 
 # enable_efa_pg_instance_config is an AWS specific option that enables one worker machineset in a placement group and with EFA Network Interface Type, other worker machinesets will be ENA Network Interface Type by default.....
@@ -628,7 +686,10 @@ case "${CLUSTER_TYPE}" in
 azure4|azure-arm64) inject_boot_diagnostics ${dir} ;;
 aws|aws-arm64|aws-usgov)
     if [[ "${SPOT_INSTANCES:-}"  == 'true' ]]; then
-      inject_spot_instance_config ${dir}
+      inject_spot_instance_config "${dir}" "workers"
+    fi
+    if [[ "${SPOT_MASTERS:-}" == 'true' ]]; then
+      inject_spot_instance_config "${dir}" "masters"
     fi
     if [[ "${ENABLE_AWS_EFA_PG_INSTANCE:-}"  == 'true' ]]; then
       enable_efa_pg_instance_config ${dir}

ci-operator/step-registry/ipi/install/install/ipi-install-install-ref.yaml

@@ -45,7 +45,10 @@ ref:
     documentation: "Specifies the logging level for terraform provider libraries."
   - name: SPOT_INSTANCES
     default: "false"
-    documentation:  "Use AWS Spot Instances for worker nodes.  Set to 'true' to opt into spot instances.  Explicitly set to 'false' to opt out.  Leave unset for the default, which may change."
+    documentation:  "Use AWS Spot Instances for *worker* nodes.  Set to 'true' to opt into spot instances.  Explicitly set to 'false' to opt out.  Leave unset for the default, which may change."
+  - name: SPOT_MASTERS
+    default: "false"
+    documentation:  "Use AWS Spot Instances for *master* nodes. Set to 'true' to opt into spot instances. Explicitly set to 'false' to opt out. Leave unset for the default, which may change. Note that spot masters are only supported when installing with a) CAPI; or b) newer installer versions (see https://github.com/openshift/installer/pull/8349). A preflight check will fail if this variable is set to 'true' for an unsupported configuration."
   - name: OPENSHIFT_INSTALL_AWS_PUBLIC_ONLY
     default: ""
     documentation: "Whether to use only public subnets for AWS. Implies no NAT Gateways. Requires a VPC to be configured prior to install."