-
Notifications
You must be signed in to change notification settings - Fork 585
[OCPCLOUD-1416] Add ControlPlaneMachineSet to Machine API group #1112
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| package v1 | ||
|
|
||
| // AWSResourceReference is a reference to a specific AWS resource by ID, ARN, or filters. | ||
| // Only one of ID, ARN or Filters may be specified. Specifying more than one will result in | ||
| // a validation error. | ||
| // +union | ||
| type AWSResourceReference struct { | ||
| // Type determines how the reference will fetch the AWS resource. | ||
| // +unionDiscriminator | ||
| // +kubebuilder:validation:Enum:="id";"arn";"filters" | ||
| // +kubebuilder:validation:Required | ||
| Type AWSResourceReferenceType `json:"type"` | ||
| // ID of resource | ||
| // +optional | ||
| ID *string `json:"id,omitempty"` | ||
| // ARN of resource | ||
| // +optional | ||
| ARN *string `json:"arn,omitempty"` | ||
| // Filters is a set of filters used to identify a resource | ||
| // +optional | ||
| Filters *[]AWSResourceFilter `json:"filters,omitempty"` | ||
| } | ||
|
|
||
| // AWSResourceReferenceType is an enumeration of different resource reference types. | ||
| type AWSResourceReferenceType string | ||
|
|
||
| const ( | ||
| // AWSIDReferenceType is a resource reference based on the object ID. | ||
| AWSIDReferenceType AWSResourceReferenceType = "id" | ||
|
|
||
| // AWSARNReferenceType is a resource reference based on the object ARN. | ||
| AWSARNReferenceType AWSResourceReferenceType = "arn" | ||
|
|
||
| // AWSFiltersReferenceType is a resource reference based on filters. | ||
| AWSFiltersReferenceType AWSResourceReferenceType = "filters" | ||
| ) | ||
|
|
||
| // AWSResourceFilter is a filter used to identify an AWS resource | ||
| type AWSResourceFilter struct { | ||
| // Name of the filter. Filter names are case-sensitive. | ||
| // +kubebuilder:validation:Required | ||
| Name string `json:"name"` | ||
| // Values includes one or more filter values. Filter values are case-sensitive. | ||
| // +optional | ||
| Values []string `json:"values,omitempty"` | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,306 @@ | ||
| package v1 | ||
|
|
||
| import ( | ||
| configv1 "github.com/openshift/api/config/v1" | ||
| machinev1beta1 "github.com/openshift/api/machine/v1beta1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| ) | ||
|
|
||
| // +genclient | ||
| // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object | ||
|
|
||
| // ControlPlaneMachineSet ensures that a specified number of control plane machine replicas are running at any given time. | ||
| // +k8s:openapi-gen=true | ||
| // +kubebuilder:resource:scope=Namespaced | ||
| // +kubebuilder:subresource:status | ||
| // +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas | ||
| // +kubebuilder:printcolumn:name="Desired",type="integer",JSONPath=".spec.replicas",description="Desired Replicas" | ||
| // +kubebuilder:printcolumn:name="Current",type="integer",JSONPath=".status.replicas",description="Current Replicas" | ||
| // +kubebuilder:printcolumn:name="Ready",type="integer",JSONPath=".status.readyReplicas",description="Ready Replicas" | ||
| // +kubebuilder:printcolumn:name="Updated",type="integer",JSONPath=".status.updatedReplicas",description="Updated Replicas" | ||
| // +kubebuilder:printcolumn:name="Unavailable",type="integer",JSONPath=".status.unavailableReplicas",description="Observed number of unavailable replicas" | ||
| // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="ControlPlaneMachineSet age" | ||
| // Compatibility level 1: Stable within a major release for a minimum of 12 months or 3 minor releases (whichever is longer). | ||
| // +openshift:compatibility-gen:level=1 | ||
| type ControlPlaneMachineSet struct { | ||
| metav1.TypeMeta `json:",inline"` | ||
| metav1.ObjectMeta `json:"metadata,omitempty"` | ||
|
|
||
| Spec ControlPlaneMachineSetSpec `json:"spec,omitempty"` | ||
| Status ControlPlaneMachineSetStatus `json:"status,omitempty"` | ||
| } | ||
|
|
||
| // ControlPlaneMachineSet represents the configuration of the ControlPlaneMachineSet. | ||
| type ControlPlaneMachineSetSpec struct { | ||
| // Replicas defines how many Control Plane Machines should be | ||
| // created by this ControlPlaneMachineSet. | ||
| // This field is immutable and cannot be changed after cluster | ||
| // installation. | ||
| // The ControlPlaneMachineSet only operates with 3 or 5 node control planes, | ||
| // 3 and 5 are the only valid values for this field. | ||
| // +kubebuilder:validation:Enum:=3;5 | ||
| // +kubebuilder:default:=3 | ||
| // +kubebuilder:validation:Required | ||
| Replicas *int32 `json:"replicas"` | ||
|
|
||
| // Strategy defines how the ControlPlaneMachineSet will update | ||
| // Machines when it detects a change to the ProviderSpec. | ||
| // +kubebuilder:default:={type: RollingUpdate} | ||
| // +optional | ||
| Strategy ControlPlaneMachineSetStrategy `json:"strategy,omitempty"` | ||
|
|
||
| // Label selector for Machines. Existing Machines selected by this | ||
| // selector will be the ones affected by this ControlPlaneMachineSet. | ||
| // It must match the template's labels. | ||
| // This field is considered immutable after creation of the resource. | ||
| // +kubebuilder:validation:Required | ||
| Selector metav1.LabelSelector `json:"selector"` | ||
|
|
||
| // Template describes the Control Plane Machines that will be created | ||
| // by this ControlPlaneMachineSet. | ||
| // +kubebuilder:validation:Required | ||
| Template ControlPlaneMachineSetTemplate `json:"template"` | ||
| } | ||
|
|
||
| // ControlPlaneMachineSetTemplate is a template used by the ControlPlaneMachineSet | ||
| // to create the Machines that it will manage in the future. | ||
| // +union | ||
| // + --- | ||
| // + This struct is a discriminated union which allows users to select the type of Machine | ||
| // + that the ControlPlaneMachineSet should create and manage. | ||
| // + For now, the only supported type is the OpenShift Machine API Machine, but in the future | ||
| // + we plan to expand this to allow other Machine types such as Cluster API Machines or a | ||
| // + future version of the Machine API Machine. | ||
| type ControlPlaneMachineSetTemplate struct { | ||
| // MachineType determines the type of Machines that should be managed by the ControlPlaneMachineSet. | ||
| // Currently, the only valid value is machine.v1beta1.machine.openshift.io. | ||
| // +unionDiscriminator | ||
| // +kubebuilder:validation:Required | ||
| MachineType ControlPlaneMachineSetMachineType `json:"machineType"` | ||
|
|
||
| // OpenShiftMachineV1Beta1Machine defines the template for creating Machines | ||
| // from the v1beta1.machine.openshift.io API group. | ||
| // +kubebuilder:validation:Required | ||
| OpenShiftMachineV1Beta1Machine *OpenShiftMachineV1Beta1MachineTemplate `json:"machines_v1beta1_machine_openshift_io,omitempty"` | ||
| } | ||
|
|
||
| // ControlPlaneMachineSetMachineType is a enumeration of valid Machine types | ||
| // supported by the ControlPlaneMachineSet. | ||
| type ControlPlaneMachineSetMachineType string | ||
|
|
||
| const ( | ||
| // OpenShiftMachineV1Beta1MachineType is the OpenShift Machine API v1beta1 Machine type. | ||
| OpenShiftMachineV1Beta1MachineType ControlPlaneMachineSetMachineType = "machines_v1beta1_machine_openshift_io" | ||
| ) | ||
|
|
||
| // OpenShiftMachineV1Beta1MachineTemplate is a template for the ControlPlaneMachineSet to create | ||
| // Machines from the v1beta1.machine.openshift.io API group. | ||
| type OpenShiftMachineV1Beta1MachineTemplate struct { | ||
| // FailureDomains is the list of failure domains (sometimes called | ||
| // availability zones) in which the ControlPlaneMachineSet should balance | ||
| // the Control Plane Machines. | ||
| // This will be merged into the ProviderSpec given in the template. | ||
| // This field is optional on platforms that do not require placement | ||
| // information, eg OpenStack. | ||
| // +optional | ||
| FailureDomains FailureDomains `json:"failureDomains,omitempty"` | ||
|
|
||
| // ObjectMeta is the standard object metadata | ||
| // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata | ||
| // Labels are required to match the ControlPlaneMachineSet selector. | ||
| // +kubebuilder:validation:Required | ||
| ObjectMeta ControlPlaneMachineSetTemplateObjectMeta `json:"metadata"` | ||
|
|
||
| // Spec contains the desired configuration of the Control Plane Machines. | ||
| // The ProviderSpec within contains platform specific details | ||
| // for creating the Control Plane Machines. | ||
| // The ProviderSe should be complete apart from the platform specific | ||
| // failure domain field. This will be overriden when the Machines | ||
| // are created based on the FailureDomains field. | ||
| // +kubebuilder:validation:Required | ||
| Spec machinev1beta1.MachineSpec `json:"spec"` | ||
| } | ||
|
|
||
| // ControlPlaneMachineSetTemplateObjectMeta is a subset of the metav1.ObjectMeta struct. | ||
| // It allows users to specify labels and annotations that will be copied onto Machines | ||
| // created from this template. | ||
| type ControlPlaneMachineSetTemplateObjectMeta struct { | ||
| // Map of string keys and values that can be used to organize and categorize | ||
| // (scope and select) objects. May match selectors of replication controllers | ||
| // and services. | ||
| // More info: http://kubernetes.io/docs/user-guide/labels | ||
| // +optional | ||
| Labels map[string]string `json:"labels,omitempty"` | ||
|
|
||
| // Annotations is an unstructured key value map stored with a resource that may be | ||
| // set by external tools to store and retrieve arbitrary metadata. They are not | ||
| // queryable and should be preserved when modifying objects. | ||
| // More info: http://kubernetes.io/docs/user-guide/annotations | ||
| // +optional | ||
| Annotations map[string]string `json:"annotations,omitempty"` | ||
| } | ||
|
|
||
| // ControlPlaneMachineSetStrategy defines the strategy for applying updates to the | ||
| // Control Plane Machines managed by the ControlPlaneMachineSet. | ||
| type ControlPlaneMachineSetStrategy struct { | ||
| // Type defines the type of update strategy that should be | ||
| // used when updating Machines owned by the ControlPlaneMachineSet. | ||
| // Valid values are "RollingUpdate" and "OnDelete". | ||
| // The current default value is "RollingUpdate". | ||
| // +kubebuilder:default:="RollingUpdate" | ||
| // +kubebuilder:validation:Enum:="RollingUpdate";"OnDelete" | ||
| // +optional | ||
| Type ControlPlaneMachineSetStrategyType `json:"type,omitempty"` | ||
|
|
||
| // This is left as a struct to allow future rolling update | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. thank you for noting this. It would not have occurred to me otherwise. |
||
| // strategy configuration to be added later. | ||
| } | ||
|
|
||
| // ControlPlaneMachineSetStrategyType is an enumeration of different update strategies | ||
| // for the Control Plane Machines. | ||
| type ControlPlaneMachineSetStrategyType string | ||
|
|
||
| const ( | ||
| // RollingUpdate is the default update strategy type for a | ||
| // ControlPlaneMachineSet. This will cause the ControlPlaneMachineSet to | ||
| // first create a new Machine and wait for this to be Ready | ||
| // before removing the Machine chosen for replacement. | ||
| RollingUpdate ControlPlaneMachineSetStrategyType = "RollingUpdate" | ||
|
|
||
| // Recreate causes the ControlPlaneMachineSet controller to first | ||
| // remove a ControlPlaneMachine before creating its | ||
| // replacement. This allows for scenarios with limited capacity | ||
| // such as baremetal environments where additional capacity to | ||
| // perform rolling updates is not available. | ||
| Recreate ControlPlaneMachineSetStrategyType = "Recreate" | ||
|
|
||
| // OnDelete causes the ControlPlaneMachineSet to only replace a | ||
| // Machine once it has been marked for deletion. This strategy | ||
| // makes the rollout of updated specifications into a manual | ||
| // process. This allows users to test new configuration on | ||
| // a single Machine without forcing the rollout of all of their | ||
| // Control Plane Machines. | ||
| OnDelete ControlPlaneMachineSetStrategyType = "OnDelete" | ||
| ) | ||
|
|
||
| // FailureDomain represents the different configurations required to spread Machines | ||
| // across failure domains on different platforms. | ||
| // +union | ||
| type FailureDomains struct { | ||
| // Platform identifies the platform for which the FailureDomain represents | ||
| // +unionDiscriminator | ||
| // +optional | ||
| Platform configv1.PlatformType `json:"platform,omitempty"` | ||
|
|
||
| // AWS configures failure domain information for the AWS platform | ||
| // +optional | ||
| AWS *[]AWSFailureDomain `json:"aws,omitempty"` | ||
|
|
||
| // Azure configures failure domain information for the Azure platform | ||
| // +optional | ||
| Azure *[]AzureFailureDomain `json:"azure,omitempty"` | ||
|
|
||
| // GCP configures failure domain information for the GCP platform | ||
| // +optional | ||
| GCP *[]GCPFailureDomain `json:"gcp,omitempty"` | ||
|
|
||
| // OpenStack configures failure domain information for the OpenStack platform | ||
| // +optional | ||
| OpenStack *[]OpenStackFailureDomain `json:"openstack,omitempty"` | ||
| } | ||
|
|
||
| // AWSFailureDomain configures failure domain information for the AWS platform | ||
| // +kubebuilder:validation:MinProperties:=1 | ||
| type AWSFailureDomain struct { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is this a "pick one" type?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No, in this struct you can have one, or the other, or both, is completely flexible and up to the user. I think in most cases users will need to use both
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Validation will need to enforce that.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you enforce "you must set at least one of these" with kubebuilder validation tags? Perhaps
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ok yep, just checked that and it works, will double check other placements as well, they all have only one field so will have to make sure they're all required |
||
| // Subnet is a reference to the subnet to use for this instance | ||
| // +optional | ||
| Subnet AWSResourceReference `json:"subnet,omitempty"` | ||
|
|
||
| // Placement configures the placement information for this instance | ||
| // +optional | ||
| Placement AWSFailureDomainPlacement `json:"placement,omitempty"` | ||
| } | ||
|
|
||
| // AWSFailureDomainPlacement configures the placement information for the AWSFailureDomain | ||
| type AWSFailureDomainPlacement struct { | ||
| // AvailabilityZone is the availability zone of the instance | ||
| // +kubebuilder:validation:Required | ||
| AvailabilityZone string `json:"availabilityZone"` | ||
| } | ||
|
|
||
| // AzureFailureDomain configures failure domain information for the Azure platform | ||
| type AzureFailureDomain struct { | ||
| // Availability Zone for the virtual machine. | ||
| // If nil, the virtual machine should be deployed to no zone | ||
| // +kubebuilder:validation:Required | ||
| Zone string `json:"zone"` | ||
| } | ||
|
|
||
| // GCPFailureDomain configures failure domain information for the GCP platform | ||
| type GCPFailureDomain struct { | ||
| // Zone is the zone in which the GCP machine provider will create the VM. | ||
| // +kubebuilder:validation:Required | ||
| Zone string `json:"zone"` | ||
| } | ||
|
|
||
| // OpenStackFailureDomain configures failure domain information for the OpenStack platform | ||
| type OpenStackFailureDomain struct { | ||
| // The availability zone from which to launch the server. | ||
| // +kubebuilder:validation:Required | ||
| AvailabilityZone string `json:"availabilityZone"` | ||
| } | ||
|
|
||
| // ControlPlaneMachineSetStatus represents the status of the ControlPlaneMachineSet CRD. | ||
| type ControlPlaneMachineSetStatus struct { | ||
| // Conditions represents the observations of the ControlPlaneMachineSet's current state. | ||
| // Known .status.conditions.type are: (TODO) | ||
| // TODO: Identify different condition types/reasons that will be needed. | ||
| // +patchMergeKey=type | ||
| // +patchStrategy=merge | ||
| // +listType=map | ||
| // +listMapKey=type | ||
| // +optional | ||
| Conditions []metav1.Condition `json:"conditions,omitempty"` | ||
|
|
||
| // ObservedGeneration is the most recent generation observed for this | ||
| // ControlPlaneMachineSet. It corresponds to the ControlPlaneMachineSets's generation, | ||
| // which is updated on mutation by the API Server. | ||
| // +optional | ||
| ObservedGeneration int64 `json:"observedGeneration,omitempty"` | ||
|
|
||
| // Replicas is the number of Control Plane Machines created by the | ||
| // ControlPlaneMachineSet controller. | ||
| // Note that during update operations this value may differ from the | ||
| // desired replica count. | ||
| // +optional | ||
| Replicas int32 `json:"replicas,omitempty"` | ||
|
|
||
| // ReadyReplicas is the number of Control Plane Machines created by the | ||
| // ControlPlaneMachineSet controller which are ready. | ||
| // +optional | ||
| ReadyReplicas int32 `json:"readyReplicas,omitempty"` | ||
|
|
||
| // UpdatedReplicas is the number of non-terminated Control Plane Machines | ||
| // created by the ControlPlaneMachineSet controller that have the desired | ||
| // provider spec. | ||
| // +optional | ||
| UpdatedReplicas int32 `json:"updatedReplicas,omitempty"` | ||
|
|
||
| // UnavailableReplicas is the number of Control Plane Machines that are | ||
| // still required before the ControlPlaneMachineSet reaches the desired | ||
| // available capacity. When this value is non-zero, the number of | ||
| // ReadyReplicas is less than the desired Replicas. | ||
| // +optional | ||
| UnavailableReplicas int32 `json:"unavailableReplicas,omitempty"` | ||
| } | ||
|
|
||
| // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object | ||
|
|
||
| // ControlPlaneMachineSetList contains a list of ControlPlaneMachineSet | ||
| // Compatibility level 1: Stable within a major release for a minimum of 12 months or 3 minor releases (whichever is longer). | ||
| // +openshift:compatibility-gen:level=1 | ||
| type ControlPlaneMachineSetList struct { | ||
| metav1.TypeMeta `json:",inline"` | ||
| metav1.ListMeta `json:"metadata,omitempty"` | ||
| Items []ControlPlaneMachineSet `json:"items"` | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
non-namespaced. If someone wants to create an externally managed control plane variant, that would be a different type so they can evolve independently.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I disagree with this statement
There are a number of arguments for making this namespaced:
I'm aware of your concern with the fact that logically there should only be one of these in an HA cluster, I believe we can resolve that by restricting it initially to a single instance in a single namespace, we already restrict Machine API to a single namespace, this is something users are familiar with
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Counterpoints in favor cluster scoped include making it impossible to let a user create one in the wrong namespace, making it impossible to accidentally grant privileges to namespace level users by using an uncareful *, making it impossible to use a different namespace that may eventually have an impact, and being at a scope that is universally recognized as privleged.
In addition, given that migration would still be one to one it could still be done with a namespaced reference.
With regard to "but we could use the same type for a different topology management mode", it seems extremely likely that the set of capabilities and control desired in self-managed versus externally managed be different as we've seen with other operators where the constraints imposed by external management are different. This is likely to lead to logically divergent schemas that you will you attempt to force into a single type.
However, in spite of all that, I am willing to let this play out if you really want it. But when you start developing divergent schemas, you will need a plan either maintain two different schemas (and thus a very confusing controller on top of it) or develop some killer validation routines that manage to avoid the cyclical re-bootstrapping problem.