diff --git a/api/hypershift/v1beta1/aws.go b/api/hypershift/v1beta1/aws.go index aea0741ef0bd..013c2f95937c 100644 --- a/api/hypershift/v1beta1/aws.go +++ b/api/hypershift/v1beta1/aws.go @@ -59,8 +59,21 @@ type AWSNodePoolPlatform struct { Placement *PlacementOptions `json:"placement,omitempty"` } +// AWSSpotMarketOptions defines configuration for AWS Spot instances +type AWSSpotMarketOptions struct { + // maxPrice defines the maximum price (USD per instance‑hour) you are willing to pay for a Spot instance. + // If omitted, the On‑Demand price is used as the ceiling. + // Example: "0.0739" + // Format: up to 10 integer digits and up to 6 fractional digits; no leading zeros unless the value is "0"; scientific notation is not allowed. + // +optional + // +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,9})(\.[0-9]{1,6})?$` + // +kubebuilder:validation:MaxLength=17 + MaxPrice *string `json:"maxPrice,omitempty"` +} + // PlacementOptions specifies the placement options for the EC2 instances. // +kubebuilder:validation:XValidation:rule="has(self.tenancy) && self.tenancy == 'host' ? !has(self.capacityReservation) : true", message="AWS Capacity Reservations cannot be used with Dedicated Hosts (tenancy 'host')" +// +kubebuilder:validation:XValidation:rule="has(self.spotMarketOptions) ? (!has(self.capacityReservation) && (!has(self.tenancy) || self.tenancy == 'default')) : true", message="spotMarketOptions is incompatible with capacityReservation and requires tenancy to be 'default' or unset (not 'dedicated' or 'host')" type PlacementOptions struct { // tenancy indicates if instance should run on shared or single-tenant hardware. // @@ -83,6 +96,12 @@ type PlacementOptions struct { // // +optional CapacityReservation *CapacityReservationOptions `json:"capacityReservation,omitempty"` + + // spotMarketOptions specifies options for using AWS Spot instances. + // When specified, instances will be launched as Spot instances with the given configuration. + // Mutually exclusive with capacityReservation, and tenancy must be unset or set to "default". + // +optional + SpotMarketOptions *AWSSpotMarketOptions `json:"spotMarketOptions,omitempty"` } // MarketType describes the market type of the CapacityReservation for an Instance. diff --git a/api/hypershift/v1beta1/zz_generated.deepcopy.go b/api/hypershift/v1beta1/zz_generated.deepcopy.go index 41a13b0ab84e..a09baf42c065 100644 --- a/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -489,6 +489,26 @@ func (in *AWSSharedVPCRolesRef) DeepCopy() *AWSSharedVPCRolesRef { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSSpotMarketOptions) DeepCopyInto(out *AWSSpotMarketOptions) { + *out = *in + if in.MaxPrice != nil { + in, out := &in.MaxPrice, &out.MaxPrice + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSSpotMarketOptions. +func (in *AWSSpotMarketOptions) DeepCopy() *AWSSpotMarketOptions { + if in == nil { + return nil + } + out := new(AWSSpotMarketOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AddressPair) DeepCopyInto(out *AddressPair) { *out = *in @@ -3114,6 +3134,11 @@ func (in *PlacementOptions) DeepCopyInto(out *PlacementOptions) { *out = new(CapacityReservationOptions) (*in).DeepCopyInto(*out) } + if in.SpotMarketOptions != nil { + in, out := &in.SpotMarketOptions, &out.SpotMarketOptions + *out = new(AWSSpotMarketOptions) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementOptions. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/AAA_ungated.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/AAA_ungated.yaml index 16ed60c7d50d..40d6aec71081 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/AAA_ungated.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/AAA_ungated.yaml @@ -484,6 +484,22 @@ spec: requires a Capacity Reservation ID rule: 'has(self.marketType) && self.marketType == ''CapacityBlocks'' ? has(self.id) : true' + spotMarketOptions: + description: |- + spotMarketOptions specifies options for using AWS Spot instances. + When specified, instances will be launched as Spot instances with the given configuration. + Mutually exclusive with capacityReservation, and tenancy must be unset or set to "default". + properties: + maxPrice: + description: |- + maxPrice defines the maximum price (USD per instance‑hour) you are willing to pay for a Spot instance. + If omitted, the On‑Demand price is used as the ceiling. + Example: "0.0739" + Format: up to 10 integer digits and up to 6 fractional digits; no leading zeros unless the value is "0"; scientific notation is not allowed. + maxLength: 17 + pattern: ^(0|[1-9][0-9]{0,9})(\.[0-9]{1,6})?$ + type: string + type: object tenancy: description: |- tenancy indicates if instance should run on shared or single-tenant hardware. @@ -506,6 +522,12 @@ spec: Hosts (tenancy 'host') rule: 'has(self.tenancy) && self.tenancy == ''host'' ? !has(self.capacityReservation) : true' + - message: spotMarketOptions is incompatible with capacityReservation + and requires tenancy to be 'default' or unset (not 'dedicated' + or 'host') + rule: 'has(self.spotMarketOptions) ? (!has(self.capacityReservation) + && (!has(self.tenancy) || self.tenancy == ''default'')) + : true' resourceTags: description: |- resourceTags is an optional list of additional tags to apply to AWS node diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/OpenStack.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/OpenStack.yaml index 5b02e316ca55..2009add3f369 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/OpenStack.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/OpenStack.yaml @@ -484,6 +484,22 @@ spec: requires a Capacity Reservation ID rule: 'has(self.marketType) && self.marketType == ''CapacityBlocks'' ? has(self.id) : true' + spotMarketOptions: + description: |- + spotMarketOptions specifies options for using AWS Spot instances. + When specified, instances will be launched as Spot instances with the given configuration. + Mutually exclusive with capacityReservation, and tenancy must be unset or set to "default". + properties: + maxPrice: + description: |- + maxPrice defines the maximum price (USD per instance‑hour) you are willing to pay for a Spot instance. + If omitted, the On‑Demand price is used as the ceiling. + Example: "0.0739" + Format: up to 10 integer digits and up to 6 fractional digits; no leading zeros unless the value is "0"; scientific notation is not allowed. + maxLength: 17 + pattern: ^(0|[1-9][0-9]{0,9})(\.[0-9]{1,6})?$ + type: string + type: object tenancy: description: |- tenancy indicates if instance should run on shared or single-tenant hardware. @@ -506,6 +522,12 @@ spec: Hosts (tenancy 'host') rule: 'has(self.tenancy) && self.tenancy == ''host'' ? !has(self.capacityReservation) : true' + - message: spotMarketOptions is incompatible with capacityReservation + and requires tenancy to be 'default' or unset (not 'dedicated' + or 'host') + rule: 'has(self.spotMarketOptions) ? (!has(self.capacityReservation) + && (!has(self.tenancy) || self.tenancy == ''default'')) + : true' resourceTags: description: |- resourceTags is an optional list of additional tags to apply to AWS node diff --git a/client/applyconfiguration/hypershift/v1beta1/awsspotmarketoptions.go b/client/applyconfiguration/hypershift/v1beta1/awsspotmarketoptions.go new file mode 100644 index 000000000000..ada7165a350c --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/awsspotmarketoptions.go @@ -0,0 +1,38 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +// AWSSpotMarketOptionsApplyConfiguration represents a declarative configuration of the AWSSpotMarketOptions type for use +// with apply. +type AWSSpotMarketOptionsApplyConfiguration struct { + MaxPrice *string `json:"maxPrice,omitempty"` +} + +// AWSSpotMarketOptionsApplyConfiguration constructs a declarative configuration of the AWSSpotMarketOptions type for use with +// apply. +func AWSSpotMarketOptions() *AWSSpotMarketOptionsApplyConfiguration { + return &AWSSpotMarketOptionsApplyConfiguration{} +} + +// WithMaxPrice sets the MaxPrice field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the MaxPrice field is set to the value of the last call. +func (b *AWSSpotMarketOptionsApplyConfiguration) WithMaxPrice(value string) *AWSSpotMarketOptionsApplyConfiguration { + b.MaxPrice = &value + return b +} diff --git a/client/applyconfiguration/hypershift/v1beta1/placementoptions.go b/client/applyconfiguration/hypershift/v1beta1/placementoptions.go index 43d5cc98dc87..aab1defde070 100644 --- a/client/applyconfiguration/hypershift/v1beta1/placementoptions.go +++ b/client/applyconfiguration/hypershift/v1beta1/placementoptions.go @@ -22,6 +22,7 @@ package v1beta1 type PlacementOptionsApplyConfiguration struct { Tenancy *string `json:"tenancy,omitempty"` CapacityReservation *CapacityReservationOptionsApplyConfiguration `json:"capacityReservation,omitempty"` + SpotMarketOptions *AWSSpotMarketOptionsApplyConfiguration `json:"spotMarketOptions,omitempty"` } // PlacementOptionsApplyConfiguration constructs a declarative configuration of the PlacementOptions type for use with @@ -45,3 +46,11 @@ func (b *PlacementOptionsApplyConfiguration) WithCapacityReservation(value *Capa b.CapacityReservation = value return b } + +// WithSpotMarketOptions sets the SpotMarketOptions field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the SpotMarketOptions field is set to the value of the last call. +func (b *PlacementOptionsApplyConfiguration) WithSpotMarketOptions(value *AWSSpotMarketOptionsApplyConfiguration) *PlacementOptionsApplyConfiguration { + b.SpotMarketOptions = value + return b +} diff --git a/client/applyconfiguration/utils.go b/client/applyconfiguration/utils.go index b670f9efab1d..37e4a9cb346f 100644 --- a/client/applyconfiguration/utils.go +++ b/client/applyconfiguration/utils.go @@ -89,6 +89,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &hypershiftv1beta1.AWSSharedVPCApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("AWSSharedVPCRolesRef"): return &hypershiftv1beta1.AWSSharedVPCRolesRefApplyConfiguration{} + case v1beta1.SchemeGroupVersion.WithKind("AWSSpotMarketOptions"): + return &hypershiftv1beta1.AWSSpotMarketOptionsApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("AzureAuthenticationConfiguration"): return &hypershiftv1beta1.AzureAuthenticationConfigurationApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("AzureKMSKey"): diff --git a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/nodepools-CustomNoUpgrade.crd.yaml b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/nodepools-CustomNoUpgrade.crd.yaml index 0423ac0621cf..018f97da78e2 100644 --- a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/nodepools-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/nodepools-CustomNoUpgrade.crd.yaml @@ -487,6 +487,22 @@ spec: requires a Capacity Reservation ID rule: 'has(self.marketType) && self.marketType == ''CapacityBlocks'' ? has(self.id) : true' + spotMarketOptions: + description: |- + spotMarketOptions specifies options for using AWS Spot instances. + When specified, instances will be launched as Spot instances with the given configuration. + Mutually exclusive with capacityReservation, and tenancy must be unset or set to "default". + properties: + maxPrice: + description: |- + maxPrice defines the maximum price (USD per instance‑hour) you are willing to pay for a Spot instance. + If omitted, the On‑Demand price is used as the ceiling. + Example: "0.0739" + Format: up to 10 integer digits and up to 6 fractional digits; no leading zeros unless the value is "0"; scientific notation is not allowed. + maxLength: 17 + pattern: ^(0|[1-9][0-9]{0,9})(\.[0-9]{1,6})?$ + type: string + type: object tenancy: description: |- tenancy indicates if instance should run on shared or single-tenant hardware. @@ -509,6 +525,12 @@ spec: Hosts (tenancy 'host') rule: 'has(self.tenancy) && self.tenancy == ''host'' ? !has(self.capacityReservation) : true' + - message: spotMarketOptions is incompatible with capacityReservation + and requires tenancy to be 'default' or unset (not 'dedicated' + or 'host') + rule: 'has(self.spotMarketOptions) ? (!has(self.capacityReservation) + && (!has(self.tenancy) || self.tenancy == ''default'')) + : true' resourceTags: description: |- resourceTags is an optional list of additional tags to apply to AWS node diff --git a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/nodepools-Default.crd.yaml b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/nodepools-Default.crd.yaml index 5ab729d41251..f350b77149ff 100644 --- a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/nodepools-Default.crd.yaml +++ b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/nodepools-Default.crd.yaml @@ -487,6 +487,22 @@ spec: requires a Capacity Reservation ID rule: 'has(self.marketType) && self.marketType == ''CapacityBlocks'' ? has(self.id) : true' + spotMarketOptions: + description: |- + spotMarketOptions specifies options for using AWS Spot instances. + When specified, instances will be launched as Spot instances with the given configuration. + Mutually exclusive with capacityReservation, and tenancy must be unset or set to "default". + properties: + maxPrice: + description: |- + maxPrice defines the maximum price (USD per instance‑hour) you are willing to pay for a Spot instance. + If omitted, the On‑Demand price is used as the ceiling. + Example: "0.0739" + Format: up to 10 integer digits and up to 6 fractional digits; no leading zeros unless the value is "0"; scientific notation is not allowed. + maxLength: 17 + pattern: ^(0|[1-9][0-9]{0,9})(\.[0-9]{1,6})?$ + type: string + type: object tenancy: description: |- tenancy indicates if instance should run on shared or single-tenant hardware. @@ -509,6 +525,12 @@ spec: Hosts (tenancy 'host') rule: 'has(self.tenancy) && self.tenancy == ''host'' ? !has(self.capacityReservation) : true' + - message: spotMarketOptions is incompatible with capacityReservation + and requires tenancy to be 'default' or unset (not 'dedicated' + or 'host') + rule: 'has(self.spotMarketOptions) ? (!has(self.capacityReservation) + && (!has(self.tenancy) || self.tenancy == ''default'')) + : true' resourceTags: description: |- resourceTags is an optional list of additional tags to apply to AWS node diff --git a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/nodepools-TechPreviewNoUpgrade.crd.yaml b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/nodepools-TechPreviewNoUpgrade.crd.yaml index 74e61ab4dd00..061c64454195 100644 --- a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/nodepools-TechPreviewNoUpgrade.crd.yaml +++ b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/nodepools-TechPreviewNoUpgrade.crd.yaml @@ -487,6 +487,22 @@ spec: requires a Capacity Reservation ID rule: 'has(self.marketType) && self.marketType == ''CapacityBlocks'' ? has(self.id) : true' + spotMarketOptions: + description: |- + spotMarketOptions specifies options for using AWS Spot instances. + When specified, instances will be launched as Spot instances with the given configuration. + Mutually exclusive with capacityReservation, and tenancy must be unset or set to "default". + properties: + maxPrice: + description: |- + maxPrice defines the maximum price (USD per instance‑hour) you are willing to pay for a Spot instance. + If omitted, the On‑Demand price is used as the ceiling. + Example: "0.0739" + Format: up to 10 integer digits and up to 6 fractional digits; no leading zeros unless the value is "0"; scientific notation is not allowed. + maxLength: 17 + pattern: ^(0|[1-9][0-9]{0,9})(\.[0-9]{1,6})?$ + type: string + type: object tenancy: description: |- tenancy indicates if instance should run on shared or single-tenant hardware. @@ -509,6 +525,12 @@ spec: Hosts (tenancy 'host') rule: 'has(self.tenancy) && self.tenancy == ''host'' ? !has(self.capacityReservation) : true' + - message: spotMarketOptions is incompatible with capacityReservation + and requires tenancy to be 'default' or unset (not 'dedicated' + or 'host') + rule: 'has(self.spotMarketOptions) ? (!has(self.capacityReservation) + && (!has(self.tenancy) || self.tenancy == ''default'')) + : true' resourceTags: description: |- resourceTags is an optional list of additional tags to apply to AWS node diff --git a/docs/content/how-to/automated-machine-management/aws-spot-instances.md b/docs/content/how-to/automated-machine-management/aws-spot-instances.md new file mode 100644 index 000000000000..43b1d1c2b5f8 --- /dev/null +++ b/docs/content/how-to/automated-machine-management/aws-spot-instances.md @@ -0,0 +1,221 @@ +--- +title: Manage AWS Spot instances in NodePools +--- + +AWS Spot instances allow you to run node pools using spare EC2 capacity at significantly reduced costs compared to On-Demand pricing. HyperShift supports configuring node pools to use AWS Spot instances through the `spotMarketOptions` field in the node pool specification. + +This guide demonstrates how to configure and manage NodePools with AWS Spot instances in an existing hosted cluster. + +## Prerequisites + +- An existing HyperShift hosted cluster on AWS +- Access to the management cluster where the NodePool resources are created +- Ensure that the AWS service-linked role for Spot is enabled in the account where the hosted cluster is installed. This is a one-time setup per account. + - You can verify if the role already exists using the following command: + ```sh + aws iam get-role --role-name AWSServiceRoleForEC2Spot + ``` + - If the role does not exist, create it with: + ```sh + aws iam create-service-linked-role --aws-service-name spot.amazonaws.com + ``` + +## Configure NodePool with Spot instances + +### Option 1: Create a new NodePool with Spot instances + +Create a new NodePool configured to use AWS Spot instances: + +```sh +cat << EOF | oc apply -f - +apiVersion: hypershift.openshift.io/v1beta1 +kind: NodePool +metadata: + name: spot-nodepool + namespace: clusters +spec: + clusterName: + replicas: 2 + management: + autoRepair: true + upgradeType: Replace + platform: + aws: + instanceType: m5.large + subnet: + id: subnet-xxxxxxxxx # Replace with your subnet ID + placement: + spotMarketOptions: + maxPrice: "0.10" # Maximum price per hour in USD (optional) + release: + image: +EOF +``` + +### Option 2: Update existing NodePool to use Spot instances + +You can also modify an existing NodePool to use Spot instances: + +```sh +oc patch nodepool -n clusters --type='merge' -p=' +{ + "spec": { + "platform": { + "aws": { + "placement": { + "spotMarketOptions": { + "maxPrice": "0.10" + } + } + } + } + } +}' +``` + +### Option 3: Remove Spot configuration from NodePool + +To convert a Spot instance NodePool back to On-Demand instances: + +```sh +oc patch nodepool -n clusters --type='json' -p=' +[ + { + "op": "remove", + "path": "/spec/platform/aws/placement/spotMarketOptions" + } +]' +``` + +## Spot instance configuration options + +The `spotMarketOptions` field supports the following configuration: + +- **maxPrice** (optional): The maximum price per hour that you're willing to pay for a Spot instance, specified in USD. If omitted, the On-Demand price is used as the maximum price. + +### Compatibility constraints + +Spot instances have the following compatibility requirements: + +- **Not compatible** with `spec.platform.aws.placement.capacityReservation` - Spot instances cannot use capacity reservations +- **Requires** `spec.platform.aws.placement.tenancy` to be `default` - Spot instances are not supported with `dedicated` or `host` tenancy + +These constraints are enforced through validation rules. Attempting to create a NodePool with incompatible configurations will result in a validation error. + +## Understanding Spot instance behavior + +### Cost savings + +AWS Spot instances can provide significant cost savings, often 50-90% less than On-Demand pricing, depending on the instance type and availability. + +### Availability and interruptions + +- Spot instances are subject to interruption when AWS needs the capacity back for On-Demand customers +- Instances receive a 2-minute warning before termination +- HyperShift automatically handles instance replacement when Spot instances are terminated through the NodePool's `autoRepair` functionality + +### Automatic instance replacement + +When a Spot instance is terminated: + +1. The Kubernetes node becomes `NotReady` +2. The NodePool controller detects the failed node +3. If `autoRepair: true` is set, a replacement instance is automatically requested +4. Pods are automatically rescheduled to available nodes + +## Best practices + +### 1. Set appropriate max price + +Consider setting a maximum price to control costs, but be aware that setting it too low may result in frequent interruptions: + +```yaml +spotMarketOptions: + maxPrice: "0.15" # Set based on your cost tolerance +``` + +### 2. Use diverse instance types + +Consider using multiple NodePools with different instance types to increase availability and reduce the likelihood of simultaneous interruptions across all nodes. + +### 3. Design for fault tolerance + +Ensure your applications can tolerate node interruptions: + +- Use appropriate pod disruption budgets +- Configure adequate replica counts for critical workloads +- Implement proper readiness and liveness probes +- Design stateless applications when possible + +### 4. Enable auto-repair + +Always enable `autoRepair` in your NodePool management configuration to ensure automatic replacement of interrupted instances: + +```yaml +management: + autoRepair: true + upgradeType: Replace +``` + +### 5. Monitor costs and interruptions + +- Regularly monitor your Spot instance usage and costs through the AWS console +- Set up CloudWatch alarms for Spot instance interruptions +- Track NodePool events to understand interruption patterns + +## Verification + +Verify that your nodes are running as Spot instances: + +```sh +# Check the NodePool status +oc get nodepool -n clusters + +# Check the nodes in the hosted cluster +oc get nodes -o wide + +# Verify instance types and Spot status in AWS console or CLI +aws ec2 describe-instances --region --filters "Name=instance-lifecycle,Values=spot" --query 'Reservations[].Instances[].{InstanceId:InstanceId,InstanceType:InstanceType,SpotInstanceRequestId:SpotInstanceRequestId,State:State.Name}' +``` + +## Troubleshooting + +### Spot instance unavailability + +If Spot instances are frequently unavailable: + +1. Consider increasing your maximum price +2. Try different instance types or sizes +3. Use multiple availability zones +4. Check AWS Spot instance pricing history and availability + +### Node replacement issues + +If nodes are not being replaced after Spot interruptions: + +1. Check the NodePool status: `oc describe nodepool -n clusters` +2. Verify that `autoRepair` is enabled in the NodePool management configuration +3. Check the HyperShift operator logs for any errors +4. Ensure the AWS service-linked role for Spot exists in your account + +### Validation errors + +If you encounter validation errors when creating Spot NodePools: + +1. Ensure you're not using dedicated or host tenancy with Spot instances +2. Remove any capacity reservation configuration when using Spot instances +3. Check that your `maxPrice` format is valid (numeric string in USD) + +### Monitoring Spot interruptions + +Monitor Spot instance interruption events: + +```sh +# Check NodePool events +oc describe nodepool -n clusters + +# Check for node replacement events +oc get events -n clusters --field-selector involvedObject.kind=NodePool +``` + +For more troubleshooting information, see the [general troubleshooting guide](../troubleshooting-general.md). \ No newline at end of file diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index 6d01e972cdbc..8b62644b6551 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -2530,6 +2530,39 @@ Example: +###AWSSpotMarketOptions { #hypershift.openshift.io/v1beta1.AWSSpotMarketOptions } +

+(Appears on: +PlacementOptions) +

+

+

AWSSpotMarketOptions defines configuration for AWS Spot instances

+

+ + + + + + + + + + + + + +
FieldDescription
+maxPrice
+ +string + +		 +(Optional) +

maxPrice defines the maximum price (USD per instance‑hour) you are willing to pay for a Spot instance. +If omitted, the On‑Demand price is used as the ceiling. +Example: “0.0739” +Format: up to 10 integer digits and up to 6 fractional digits; no leading zeros unless the value is “0”; scientific notation is not allowed.

+
###AddressPair { #hypershift.openshift.io/v1beta1.AddressPair }

(Appears on: @@ -10436,6 +10469,22 @@ CapacityReservationOptions do not support Capacity Reservations. Compatible with “default” and “dedicated” tenancy.

+ + +spotMarketOptions
+ + +AWSSpotMarketOptions + + + + +(Optional) +

spotMarketOptions specifies options for using AWS Spot instances. +When specified, instances will be launched as Spot instances with the given configuration. +Mutually exclusive with capacityReservation, and tenancy must be unset or set to “default”.

+ + ###PlatformSpec { #hypershift.openshift.io/v1beta1.PlatformSpec } diff --git a/hypershift-operator/controllers/nodepool/aws.go b/hypershift-operator/controllers/nodepool/aws.go index 3334d655b61f..d2113a9144da 100644 --- a/hypershift-operator/controllers/nodepool/aws.go +++ b/hypershift-operator/controllers/nodepool/aws.go @@ -7,6 +7,8 @@ import ( hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/support/releaseinfo" + "github.com/aws/aws-sdk-go/service/ec2" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/errors" @@ -164,6 +166,13 @@ func awsMachineTemplateSpec(infraName string, hostedCluster *hyperv1.HostedClust } } + if nodePool.Spec.Platform.AWS.Placement != nil && nodePool.Spec.Platform.AWS.Placement.SpotMarketOptions != nil { + awsMachineTemplateSpec.Template.Spec.SpotMarketOptions = &capiaws.SpotMarketOptions{ + MaxPrice: nodePool.Spec.Platform.AWS.Placement.SpotMarketOptions.MaxPrice, + } + awsMachineTemplateSpec.Template.Spec.MarketType = capiaws.MarketTypeSpot + } + if hostedCluster.Annotations[hyperv1.AWSMachinePublicIPs] == "true" { awsMachineTemplateSpec.Template.Spec.PublicIP = ptr.To(true) } @@ -300,6 +309,9 @@ func (r *NodePoolReconciler) setAWSConditions(ctx context.Context, nodePool *hyp } func (r NodePoolReconciler) validateAWSPlatformConfig(ctx context.Context, nodePool *hyperv1.NodePool, hc *hyperv1.HostedCluster, oldCondition *hyperv1.NodePoolCondition) error { + if nodePool.Spec.Platform.AWS == nil { + return fmt.Errorf("aws platform not populated") + } if nodePool.Spec.Platform.AWS.Placement != nil && nodePool.Spec.Platform.AWS.Placement.CapacityReservation != nil { pullSecretBytes, err := r.getPullSecretBytes(ctx, hc) if err != nil { @@ -315,5 +327,14 @@ func (r NodePoolReconciler) validateAWSPlatformConfig(ctx context.Context, nodeP } } + if nodePool.Spec.Platform.AWS.Placement != nil && nodePool.Spec.Platform.AWS.Placement.SpotMarketOptions != nil { + if nodePool.Spec.Platform.AWS.Placement.Tenancy != "" && nodePool.Spec.Platform.AWS.Placement.Tenancy != ec2.TenancyDefault { + return fmt.Errorf("spotMarketOptions is incompatible with capacityReservation and requires tenancy to be 'default' or unset (not 'dedicated' or 'host')") + } + if nodePool.Spec.Platform.AWS.Placement.CapacityReservation != nil { + return fmt.Errorf("spotMarketOptions is incompatible with capacityReservation and requires tenancy to be 'default' or unset (not 'dedicated' or 'host')") + } + } + return nil } diff --git a/hypershift-operator/controllers/nodepool/aws_test.go b/hypershift-operator/controllers/nodepool/aws_test.go index b6dc962f1f04..11116af4526e 100644 --- a/hypershift-operator/controllers/nodepool/aws_test.go +++ b/hypershift-operator/controllers/nodepool/aws_test.go @@ -12,6 +12,8 @@ import ( configv1 "github.com/openshift/api/config/v1" + "github.com/aws/aws-sdk-go/service/ec2" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -166,6 +168,36 @@ func TestAWSMachineTemplateSpec(t *testing.T) { tmpl.Spec.Template.Spec.InstanceMetadataOptions.HTTPTokens = capiaws.HTTPTokensStateRequired }), }, + { + name: "SpotMarketOptions with MaxPrice", + nodePool: hyperv1.NodePoolSpec{Platform: hyperv1.NodePoolPlatform{AWS: &hyperv1.AWSNodePoolPlatform{ + AMI: amiName, + Placement: &hyperv1.PlacementOptions{ + SpotMarketOptions: &hyperv1.AWSSpotMarketOptions{ + MaxPrice: ptr.To("0.50"), + }, + }, + }}}, + expected: defaultAWSMachineTemplate(func(tmpl *capiaws.AWSMachineTemplate) { + tmpl.Spec.Template.Spec.SpotMarketOptions = &capiaws.SpotMarketOptions{ + MaxPrice: ptr.To("0.50"), + } + tmpl.Spec.Template.Spec.MarketType = capiaws.MarketTypeSpot + }), + }, + { + name: "SpotMarketOptions without MaxPrice", + nodePool: hyperv1.NodePoolSpec{Platform: hyperv1.NodePoolPlatform{AWS: &hyperv1.AWSNodePoolPlatform{ + AMI: amiName, + Placement: &hyperv1.PlacementOptions{ + SpotMarketOptions: &hyperv1.AWSSpotMarketOptions{}, + }, + }}}, + expected: defaultAWSMachineTemplate(func(tmpl *capiaws.AWSMachineTemplate) { + tmpl.Spec.Template.Spec.SpotMarketOptions = &capiaws.SpotMarketOptions{} + tmpl.Spec.Template.Spec.MarketType = capiaws.MarketTypeSpot + }), + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -495,7 +527,7 @@ func TestValidateAWSPlatformConfig(t *testing.T) { }, } - capacityReservationID := "cr-fakeID" + capacityReservationID := "cr-0123456789abcdef0" pullSecret := fakePullSecret(hostedcluster) fakeClient := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(pullSecret).Build() @@ -503,13 +535,134 @@ func TestValidateAWSPlatformConfig(t *testing.T) { testCases := []struct { name string hostedClusterVersion string + setupNodePool func() *hyperv1.NodePool oldCondition *hyperv1.NodePoolCondition expectedError string }{ { name: "If hostedCluster < 4.19 it should fail", hostedClusterVersion: "4.18.0", - expectedError: "capacityReservation is only supported on 4.19+ clusters", + setupNodePool: func() *hyperv1.NodePool { + return &hyperv1.NodePool{ + Spec: hyperv1.NodePoolSpec{ + Platform: hyperv1.NodePoolPlatform{ + AWS: &hyperv1.AWSNodePoolPlatform{ + Placement: &hyperv1.PlacementOptions{ + CapacityReservation: &hyperv1.CapacityReservationOptions{ + ID: &capacityReservationID, + }, + }, + }, + }, + }, + } + }, + expectedError: "capacityReservation is only supported on 4.19+ clusters", + }, + { + name: "SpotMarketOptions with non default tenancy should fail", + hostedClusterVersion: "4.19.0", + setupNodePool: func() *hyperv1.NodePool { + return &hyperv1.NodePool{ + Spec: hyperv1.NodePoolSpec{ + Platform: hyperv1.NodePoolPlatform{ + AWS: &hyperv1.AWSNodePoolPlatform{ + Placement: &hyperv1.PlacementOptions{ + Tenancy: ec2.TenancyDedicated, + SpotMarketOptions: &hyperv1.AWSSpotMarketOptions{ + MaxPrice: ptr.To("0.50"), + }, + }, + }, + }, + }, + } + }, + expectedError: "spotMarketOptions is incompatible with capacityReservation and requires tenancy to be 'default' or unset (not 'dedicated' or 'host')", + }, + { + name: "SpotMarketOptions with default tenancy should succeed", + hostedClusterVersion: "4.19.0", + setupNodePool: func() *hyperv1.NodePool { + return &hyperv1.NodePool{ + Spec: hyperv1.NodePoolSpec{ + Platform: hyperv1.NodePoolPlatform{ + AWS: &hyperv1.AWSNodePoolPlatform{ + Placement: &hyperv1.PlacementOptions{ + Tenancy: ec2.TenancyDefault, + SpotMarketOptions: &hyperv1.AWSSpotMarketOptions{ + MaxPrice: ptr.To("0.30"), + }, + }, + }, + }, + }, + } + }, + expectedError: "", // No error expected + }, + { + name: "SpotMarketOptions with empty tenancy should succeed", + hostedClusterVersion: "4.19.0", + setupNodePool: func() *hyperv1.NodePool { + return &hyperv1.NodePool{ + Spec: hyperv1.NodePoolSpec{ + Platform: hyperv1.NodePoolPlatform{ + AWS: &hyperv1.AWSNodePoolPlatform{ + Placement: &hyperv1.PlacementOptions{ + SpotMarketOptions: &hyperv1.AWSSpotMarketOptions{ + MaxPrice: ptr.To("0.30"), + }, + }, + }, + }, + }, + } + }, + expectedError: "", // No error expected + }, + { + name: "SpotMarketOptions with CapacityReservation should fail", + hostedClusterVersion: "4.19.0", + setupNodePool: func() *hyperv1.NodePool { + return &hyperv1.NodePool{ + Spec: hyperv1.NodePoolSpec{ + Platform: hyperv1.NodePoolPlatform{ + AWS: &hyperv1.AWSNodePoolPlatform{ + Placement: &hyperv1.PlacementOptions{ + CapacityReservation: &hyperv1.CapacityReservationOptions{ + ID: &capacityReservationID, + }, + SpotMarketOptions: &hyperv1.AWSSpotMarketOptions{ + MaxPrice: ptr.To("0.50"), + }, + }, + }, + }, + }, + } + }, + expectedError: "spotMarketOptions is incompatible with capacityReservation and requires tenancy to be 'default' or unset (not 'dedicated' or 'host')", + }, + { + name: "SpotMarketOptions without CapacityReservation should succeed", + hostedClusterVersion: "4.19.0", + setupNodePool: func() *hyperv1.NodePool { + return &hyperv1.NodePool{ + Spec: hyperv1.NodePoolSpec{ + Platform: hyperv1.NodePoolPlatform{ + AWS: &hyperv1.AWSNodePoolPlatform{ + Placement: &hyperv1.PlacementOptions{ + SpotMarketOptions: &hyperv1.AWSSpotMarketOptions{ + MaxPrice: ptr.To("0.50"), + }, + }, + }, + }, + }, + } + }, + expectedError: "", // No error expected }, } @@ -524,19 +677,7 @@ func TestValidateAWSPlatformConfig(t *testing.T) { }, } - nodePool := &hyperv1.NodePool{ - Spec: hyperv1.NodePoolSpec{ - Platform: hyperv1.NodePoolPlatform{ - AWS: &hyperv1.AWSNodePoolPlatform{ - Placement: &hyperv1.PlacementOptions{ - CapacityReservation: &hyperv1.CapacityReservationOptions{ - ID: &capacityReservationID, - }, - }, - }, - }, - }, - } + nodePool := tc.setupNodePool() reconciler := &NodePoolReconciler{ Client: fakeClient, diff --git a/test/e2e/create_cluster_test.go b/test/e2e/create_cluster_test.go index 7ab65d1f0849..916c8a13ffff 100644 --- a/test/e2e/create_cluster_test.go +++ b/test/e2e/create_cluster_test.go @@ -1792,6 +1792,77 @@ func TestOnCreateAPIUX(t *testing.T) { }, expectedErrorSubstring: "", }, + { + name: "when spot instances are configured with dedicated tenancy it should fail", + mutateInput: func(np *hyperv1.NodePool) { + np.Spec.Platform.AWS.Placement = &hyperv1.PlacementOptions{ + Tenancy: "dedicated", + SpotMarketOptions: &hyperv1.AWSSpotMarketOptions{ + MaxPrice: ptr.To("0.50"), + }, + } + }, + expectedErrorSubstring: "spotMarketOptions is incompatible with capacityReservation and requires tenancy to be 'default' or unset (not 'dedicated' or 'host')", + }, + { + name: "when spot instances are configured with host tenancy it should fail", + mutateInput: func(np *hyperv1.NodePool) { + np.Spec.Platform.AWS.Placement = &hyperv1.PlacementOptions{ + Tenancy: "host", + SpotMarketOptions: &hyperv1.AWSSpotMarketOptions{ + MaxPrice: ptr.To("0.75"), + }, + } + }, + expectedErrorSubstring: "spotMarketOptions is incompatible with capacityReservation and requires tenancy to be 'default' or unset (not 'dedicated' or 'host')", + }, + { + name: "when spot instances are configured with capacity reservation it should fail", + mutateInput: func(np *hyperv1.NodePool) { + np.Spec.Platform.AWS.Placement = &hyperv1.PlacementOptions{ + Tenancy: "default", + CapacityReservation: &hyperv1.CapacityReservationOptions{ + ID: ptr.To("cr-0123456789abcdef0"), + }, + SpotMarketOptions: &hyperv1.AWSSpotMarketOptions{ + MaxPrice: ptr.To("0.50"), + }, + } + }, + expectedErrorSubstring: "spotMarketOptions is incompatible with capacityReservation and requires tenancy to be 'default' or unset (not 'dedicated' or 'host')", + }, + { + name: "when spot instances are configured with both dedicated tenancy and capacity reservation it should fail", + mutateInput: func(np *hyperv1.NodePool) { + np.Spec.Platform.AWS.Placement = &hyperv1.PlacementOptions{ + Tenancy: "dedicated", + CapacityReservation: &hyperv1.CapacityReservationOptions{ + ID: ptr.To("cr-0123456789abcdef0"), + MarketType: hyperv1.MarketTypeOnDemand, + }, + SpotMarketOptions: &hyperv1.AWSSpotMarketOptions{ + MaxPrice: ptr.To("0.50"), + }, + } + }, + expectedErrorSubstring: "spotMarketOptions is incompatible with capacityReservation and requires tenancy to be 'default' or unset (not 'dedicated' or 'host')", + }, + { + name: "when spot instances are configured with capacity blocks it should fail", + mutateInput: func(np *hyperv1.NodePool) { + np.Spec.Platform.AWS.Placement = &hyperv1.PlacementOptions{ + Tenancy: "default", + CapacityReservation: &hyperv1.CapacityReservationOptions{ + ID: ptr.To("cr-0123456789abcdef0"), + MarketType: hyperv1.MarketTypeCapacityBlock, + }, + SpotMarketOptions: &hyperv1.AWSSpotMarketOptions{ + MaxPrice: ptr.To("2.00"), + }, + } + }, + expectedErrorSubstring: "spotMarketOptions is incompatible with capacityReservation and requires tenancy to be 'default' or unset (not 'dedicated' or 'host')", + }, }, }, } diff --git a/test/e2e/nodepool_spotmarketoptions_test.go b/test/e2e/nodepool_spotmarketoptions_test.go new file mode 100644 index 000000000000..3f8c068c56e5 --- /dev/null +++ b/test/e2e/nodepool_spotmarketoptions_test.go @@ -0,0 +1,73 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "context" + "testing" + + . "github.com/onsi/gomega" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +// SpotMarketOptionsSuccessTest tests valid SpotMarketOptions configurations +type SpotMarketOptionsSuccessTest struct { + DummyInfraSetup + ctx context.Context + hostedCluster *hyperv1.HostedCluster +} + +func NewSpotMarketOptionsSuccessTest(ctx context.Context, hostedCluster *hyperv1.HostedCluster) *SpotMarketOptionsSuccessTest { + return &SpotMarketOptionsSuccessTest{ + ctx: ctx, + hostedCluster: hostedCluster, + } +} + +func (test *SpotMarketOptionsSuccessTest) Setup(t *testing.T) { + if globalOpts.Platform != hyperv1.AWSPlatform { + t.Skip("test only supported on platform AWS") + } +} + +func (test *SpotMarketOptionsSuccessTest) BuildNodePoolManifest(defaultNodepool hyperv1.NodePool) (*hyperv1.NodePool, error) { + nodePool := &hyperv1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: test.hostedCluster.Name + "-spot-success-test", + Namespace: test.hostedCluster.Namespace, + }, + } + defaultNodepool.Spec.DeepCopyInto(&nodePool.Spec) + nodePool.Spec.Replicas = &oneReplicas + if nodePool.Spec.Platform.AWS == nil { + nodePool.Spec.Platform.AWS = &hyperv1.AWSNodePoolPlatform{} + } + nodePool.Spec.Platform.AWS.Placement = &hyperv1.PlacementOptions{ + Tenancy: "default", + SpotMarketOptions: &hyperv1.AWSSpotMarketOptions{ + MaxPrice: ptr.To("0.50"), + }, + } + return nodePool, nil +} + +func (test *SpotMarketOptionsSuccessTest) Run(t *testing.T, nodePool hyperv1.NodePool, nodes []corev1.Node) { + g := NewWithT(t) + + // Verify the NodePool was created successfully with Spot configuration + g.Expect(nodePool.Spec.Platform.AWS.Placement.SpotMarketOptions).ToNot(BeNil()) + g.Expect(nodePool.Spec.Platform.AWS.Placement.SpotMarketOptions.MaxPrice).To(Equal(ptr.To("0.50"))) + g.Expect(nodePool.Spec.Platform.AWS.Placement.Tenancy).To(Equal("default")) + + t.Logf("Successfully created NodePool with Spot instances. MaxPrice: %v, Tenancy: %v", + *nodePool.Spec.Platform.AWS.Placement.SpotMarketOptions.MaxPrice, + nodePool.Spec.Platform.AWS.Placement.Tenancy) + + // Verify nodes were created + g.Expect(nodes).ToNot(BeEmpty()) + t.Logf("Successfully provisioned %d Spot instance nodes", len(nodes)) +} diff --git a/test/e2e/nodepool_test.go b/test/e2e/nodepool_test.go index 1b09cc22ac83..a9c0373ab086 100644 --- a/test/e2e/nodepool_test.go +++ b/test/e2e/nodepool_test.go @@ -47,6 +47,10 @@ func TestNodePool(t *testing.T) { { build: func(ctx context.Context, mgtClient crclient.Client, hostedCluster *hyperv1.HostedCluster, hostedClusterClient crclient.Client, clusterOpts e2eutil.PlatformAgnosticOptions) []NodePoolTestCase { return []NodePoolTestCase{ + { + name: "TestAWSSpotMarketOptionsSuccess", + test: NewSpotMarketOptionsSuccessTest(ctx, hostedCluster), + }, { name: "TestKMSRootVolumeEncryption", test: NewKMSRootVolumeTest(ctx, hostedCluster, clusterOpts), diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/aws.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/aws.go index aea0741ef0bd..013c2f95937c 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/aws.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/aws.go @@ -59,8 +59,21 @@ type AWSNodePoolPlatform struct { Placement *PlacementOptions `json:"placement,omitempty"` } +// AWSSpotMarketOptions defines configuration for AWS Spot instances +type AWSSpotMarketOptions struct { + // maxPrice defines the maximum price (USD per instance‑hour) you are willing to pay for a Spot instance. + // If omitted, the On‑Demand price is used as the ceiling. + // Example: "0.0739" + // Format: up to 10 integer digits and up to 6 fractional digits; no leading zeros unless the value is "0"; scientific notation is not allowed. + // +optional + // +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,9})(\.[0-9]{1,6})?$` + // +kubebuilder:validation:MaxLength=17 + MaxPrice *string `json:"maxPrice,omitempty"` +} + // PlacementOptions specifies the placement options for the EC2 instances. // +kubebuilder:validation:XValidation:rule="has(self.tenancy) && self.tenancy == 'host' ? !has(self.capacityReservation) : true", message="AWS Capacity Reservations cannot be used with Dedicated Hosts (tenancy 'host')" +// +kubebuilder:validation:XValidation:rule="has(self.spotMarketOptions) ? (!has(self.capacityReservation) && (!has(self.tenancy) || self.tenancy == 'default')) : true", message="spotMarketOptions is incompatible with capacityReservation and requires tenancy to be 'default' or unset (not 'dedicated' or 'host')" type PlacementOptions struct { // tenancy indicates if instance should run on shared or single-tenant hardware. // @@ -83,6 +96,12 @@ type PlacementOptions struct { // // +optional CapacityReservation *CapacityReservationOptions `json:"capacityReservation,omitempty"` + + // spotMarketOptions specifies options for using AWS Spot instances. + // When specified, instances will be launched as Spot instances with the given configuration. + // Mutually exclusive with capacityReservation, and tenancy must be unset or set to "default". + // +optional + SpotMarketOptions *AWSSpotMarketOptions `json:"spotMarketOptions,omitempty"` } // MarketType describes the market type of the CapacityReservation for an Instance. diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go index 41a13b0ab84e..a09baf42c065 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -489,6 +489,26 @@ func (in *AWSSharedVPCRolesRef) DeepCopy() *AWSSharedVPCRolesRef { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSSpotMarketOptions) DeepCopyInto(out *AWSSpotMarketOptions) { + *out = *in + if in.MaxPrice != nil { + in, out := &in.MaxPrice, &out.MaxPrice + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSSpotMarketOptions. +func (in *AWSSpotMarketOptions) DeepCopy() *AWSSpotMarketOptions { + if in == nil { + return nil + } + out := new(AWSSpotMarketOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AddressPair) DeepCopyInto(out *AddressPair) { *out = *in @@ -3114,6 +3134,11 @@ func (in *PlacementOptions) DeepCopyInto(out *PlacementOptions) { *out = new(CapacityReservationOptions) (*in).DeepCopyInto(*out) } + if in.SpotMarketOptions != nil { + in, out := &in.SpotMarketOptions, &out.SpotMarketOptions + *out = new(AWSSpotMarketOptions) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementOptions.