diff --git a/controlplane/kubeadm/api/v1alpha3/conversion.go b/controlplane/kubeadm/api/v1alpha3/conversion.go index f19a979f855e..9776ea265b17 100644 --- a/controlplane/kubeadm/api/v1alpha3/conversion.go +++ b/controlplane/kubeadm/api/v1alpha3/conversion.go @@ -17,18 +17,44 @@ limitations under the License. package v1alpha3 import ( + apiconversion "k8s.io/apimachinery/pkg/conversion" "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha4" + + utilconversion "sigs.k8s.io/cluster-api/util/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" ) func (src *KubeadmControlPlane) ConvertTo(destRaw conversion.Hub) error { dest := destRaw.(*v1alpha4.KubeadmControlPlane) - return Convert_v1alpha3_KubeadmControlPlane_To_v1alpha4_KubeadmControlPlane(src, dest, nil) + + if err := Convert_v1alpha3_KubeadmControlPlane_To_v1alpha4_KubeadmControlPlane(src, dest, nil); err != nil { + return err + } + + // Manually restore data. + restored := &v1alpha4.KubeadmControlPlane{} + if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok { + return err + } + + dest.Spec.RolloutStrategy = restored.Spec.RolloutStrategy + + return nil } func (dest *KubeadmControlPlane) ConvertFrom(srcRaw conversion.Hub) error { src := srcRaw.(*v1alpha4.KubeadmControlPlane) - return Convert_v1alpha4_KubeadmControlPlane_To_v1alpha3_KubeadmControlPlane(src, dest, nil) + + if err := Convert_v1alpha4_KubeadmControlPlane_To_v1alpha3_KubeadmControlPlane(src, dest, nil); err != nil { + return err + } + + // Preserve Hub data on down-conversion except for metadata + if err := utilconversion.MarshalData(src, dest); err != nil { + return err + } + + return nil } func (src *KubeadmControlPlaneList) ConvertTo(destRaw conversion.Hub) error { @@ -40,3 +66,11 @@ func (dest *KubeadmControlPlaneList) ConvertFrom(srcRaw conversion.Hub) error { src := srcRaw.(*v1alpha4.KubeadmControlPlaneList) return Convert_v1alpha4_KubeadmControlPlaneList_To_v1alpha3_KubeadmControlPlaneList(src, dest, nil) } + +func Convert_v1alpha4_KubeadmControlPlaneSpec_To_v1alpha3_KubeadmControlPlaneSpec(in *v1alpha4.KubeadmControlPlaneSpec, out *KubeadmControlPlaneSpec, s apiconversion.Scope) error { + return autoConvert_v1alpha4_KubeadmControlPlaneSpec_To_v1alpha3_KubeadmControlPlaneSpec(in, out, s) +} + +func Convert_v1alpha3_KubeadmControlPlaneSpec_To_v1alpha4_KubeadmControlPlaneSpec(in *KubeadmControlPlaneSpec, out *v1alpha4.KubeadmControlPlaneSpec, s apiconversion.Scope) error { //nolint + return autoConvert_v1alpha3_KubeadmControlPlaneSpec_To_v1alpha4_KubeadmControlPlaneSpec(in, out, s) +} diff --git a/controlplane/kubeadm/api/v1alpha3/conversion_test.go b/controlplane/kubeadm/api/v1alpha3/conversion_test.go index 2169ab16ba6a..8de6c65b0960 100644 --- a/controlplane/kubeadm/api/v1alpha3/conversion_test.go +++ b/controlplane/kubeadm/api/v1alpha3/conversion_test.go @@ -19,9 +19,12 @@ package v1alpha3 import ( "testing" + fuzz "github.com/google/gofuzz" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/runtime" + runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" + kubeadmv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1" "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha4" utilconversion "sigs.k8s.io/cluster-api/util/conversion" ) @@ -32,5 +35,25 @@ func TestFuzzyConversion(t *testing.T) { g.Expect(AddToScheme(scheme)).To(Succeed()) g.Expect(v1alpha4.AddToScheme(scheme)).To(Succeed()) - t.Run("for KubeadmControlPLane", utilconversion.FuzzTestFunc(scheme, &v1alpha4.KubeadmControlPlane{}, &KubeadmControlPlane{})) + t.Run("for KubeadmControlPLane", utilconversion.FuzzTestFunc( + scheme, &v1alpha4.KubeadmControlPlane{}, &KubeadmControlPlane{}, + func(codecs runtimeserializer.CodecFactory) []interface{} { + return []interface{}{ + // This custom function is needed when ConvertTo/ConvertFrom functions + // uses the json package to unmarshal the bootstrap token string. + // + // The Kubeadm v1beta1.BootstrapTokenString type ships with a custom + // json string representation, in particular it supplies a customized + // UnmarshalJSON function that can return an error if the string + // isn't in the correct form. + // + // This function effectively disables any fuzzing for the token by setting + // the values for ID and Secret to working alphanumeric values. + func(in *kubeadmv1.BootstrapTokenString, c fuzz.Continue) { + in.ID = "abcdef" + in.Secret = "abcdef0123456789" + }, + } + }, + )) } diff --git a/controlplane/kubeadm/api/v1alpha3/zz_generated.conversion.go b/controlplane/kubeadm/api/v1alpha3/zz_generated.conversion.go index a38ad3c9effb..b5f2b0bb36fe 100644 --- a/controlplane/kubeadm/api/v1alpha3/zz_generated.conversion.go +++ b/controlplane/kubeadm/api/v1alpha3/zz_generated.conversion.go @@ -60,23 +60,23 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*KubeadmControlPlaneSpec)(nil), (*v1alpha4.KubeadmControlPlaneSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha3_KubeadmControlPlaneSpec_To_v1alpha4_KubeadmControlPlaneSpec(a.(*KubeadmControlPlaneSpec), b.(*v1alpha4.KubeadmControlPlaneSpec), scope) + if err := s.AddGeneratedConversionFunc((*KubeadmControlPlaneStatus)(nil), (*v1alpha4.KubeadmControlPlaneStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_KubeadmControlPlaneStatus_To_v1alpha4_KubeadmControlPlaneStatus(a.(*KubeadmControlPlaneStatus), b.(*v1alpha4.KubeadmControlPlaneStatus), scope) }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha4.KubeadmControlPlaneSpec)(nil), (*KubeadmControlPlaneSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha4_KubeadmControlPlaneSpec_To_v1alpha3_KubeadmControlPlaneSpec(a.(*v1alpha4.KubeadmControlPlaneSpec), b.(*KubeadmControlPlaneSpec), scope) + if err := s.AddGeneratedConversionFunc((*v1alpha4.KubeadmControlPlaneStatus)(nil), (*KubeadmControlPlaneStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha4_KubeadmControlPlaneStatus_To_v1alpha3_KubeadmControlPlaneStatus(a.(*v1alpha4.KubeadmControlPlaneStatus), b.(*KubeadmControlPlaneStatus), scope) }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*KubeadmControlPlaneStatus)(nil), (*v1alpha4.KubeadmControlPlaneStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha3_KubeadmControlPlaneStatus_To_v1alpha4_KubeadmControlPlaneStatus(a.(*KubeadmControlPlaneStatus), b.(*v1alpha4.KubeadmControlPlaneStatus), scope) + if err := s.AddConversionFunc((*KubeadmControlPlaneSpec)(nil), (*v1alpha4.KubeadmControlPlaneSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_KubeadmControlPlaneSpec_To_v1alpha4_KubeadmControlPlaneSpec(a.(*KubeadmControlPlaneSpec), b.(*v1alpha4.KubeadmControlPlaneSpec), scope) }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha4.KubeadmControlPlaneStatus)(nil), (*KubeadmControlPlaneStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha4_KubeadmControlPlaneStatus_To_v1alpha3_KubeadmControlPlaneStatus(a.(*v1alpha4.KubeadmControlPlaneStatus), b.(*KubeadmControlPlaneStatus), scope) + if err := s.AddConversionFunc((*v1alpha4.KubeadmControlPlaneSpec)(nil), (*KubeadmControlPlaneSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha4_KubeadmControlPlaneSpec_To_v1alpha3_KubeadmControlPlaneSpec(a.(*v1alpha4.KubeadmControlPlaneSpec), b.(*KubeadmControlPlaneSpec), scope) }); err != nil { return err } @@ -169,11 +169,6 @@ func autoConvert_v1alpha3_KubeadmControlPlaneSpec_To_v1alpha4_KubeadmControlPlan return nil } -// Convert_v1alpha3_KubeadmControlPlaneSpec_To_v1alpha4_KubeadmControlPlaneSpec is an autogenerated conversion function. -func Convert_v1alpha3_KubeadmControlPlaneSpec_To_v1alpha4_KubeadmControlPlaneSpec(in *KubeadmControlPlaneSpec, out *v1alpha4.KubeadmControlPlaneSpec, s conversion.Scope) error { - return autoConvert_v1alpha3_KubeadmControlPlaneSpec_To_v1alpha4_KubeadmControlPlaneSpec(in, out, s) -} - func autoConvert_v1alpha4_KubeadmControlPlaneSpec_To_v1alpha3_KubeadmControlPlaneSpec(in *v1alpha4.KubeadmControlPlaneSpec, out *KubeadmControlPlaneSpec, s conversion.Scope) error { out.Replicas = (*int32)(unsafe.Pointer(in.Replicas)) out.Version = in.Version @@ -183,14 +178,10 @@ func autoConvert_v1alpha4_KubeadmControlPlaneSpec_To_v1alpha3_KubeadmControlPlan } out.UpgradeAfter = (*v1.Time)(unsafe.Pointer(in.UpgradeAfter)) out.NodeDrainTimeout = (*v1.Duration)(unsafe.Pointer(in.NodeDrainTimeout)) + // WARNING: in.RolloutStrategy requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha4_KubeadmControlPlaneSpec_To_v1alpha3_KubeadmControlPlaneSpec is an autogenerated conversion function. -func Convert_v1alpha4_KubeadmControlPlaneSpec_To_v1alpha3_KubeadmControlPlaneSpec(in *v1alpha4.KubeadmControlPlaneSpec, out *KubeadmControlPlaneSpec, s conversion.Scope) error { - return autoConvert_v1alpha4_KubeadmControlPlaneSpec_To_v1alpha3_KubeadmControlPlaneSpec(in, out, s) -} - func autoConvert_v1alpha3_KubeadmControlPlaneStatus_To_v1alpha4_KubeadmControlPlaneStatus(in *KubeadmControlPlaneStatus, out *v1alpha4.KubeadmControlPlaneStatus, s conversion.Scope) error { out.Selector = in.Selector out.Replicas = in.Replicas diff --git a/controlplane/kubeadm/api/v1alpha4/kubeadm_control_plane_types.go b/controlplane/kubeadm/api/v1alpha4/kubeadm_control_plane_types.go index e14c1d5d05ff..b050082d6adf 100644 --- a/controlplane/kubeadm/api/v1alpha4/kubeadm_control_plane_types.go +++ b/controlplane/kubeadm/api/v1alpha4/kubeadm_control_plane_types.go @@ -19,12 +19,21 @@ package v1alpha4 import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha4" "sigs.k8s.io/cluster-api/errors" ) +type RolloutStrategyType string + +const ( + // Replace the old control planes by new one using rolling update + // i.e. gradually scale up or down the old control planes and scale up or down the new one. + RollingUpdateStrategyType RolloutStrategyType = "RollingUpdate" +) + const ( KubeadmControlPlaneFinalizer = "kubeadm.controlplane.cluster.x-k8s.io" @@ -69,6 +78,38 @@ type KubeadmControlPlaneSpec struct { // NOTE: NodeDrainTimeout is different from `kubectl drain --timeout` // +optional NodeDrainTimeout *metav1.Duration `json:"nodeDrainTimeout,omitempty"` + + // The RolloutStrategy to use to replace control plane machines with + // new ones. + // +optional + RolloutStrategy *RolloutStrategy `json:"rolloutStrategy,omitempty"` +} + +// RolloutStrategy describes how to replace existing machines +// with new ones. +type RolloutStrategy struct { + // Type of rollout. Currently the only supported strategy is + // "RollingUpdate". + // Default is RollingUpdate. + // +optional + Type RolloutStrategyType `json:"type,omitempty"` + + // Rolling update config params. Present only if + // RolloutStrategyType = RollingUpdate. + // +optional + RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"` +} + +// RollingUpdate is used to control the desired behavior of rolling update. +type RollingUpdate struct { + // The maximum number of control planes that can be scheduled above or under the + // desired number of control planes. + // Value can be an absolute number 1 or 0. + // Defaults to 1. + // Example: when this is set to 1, the control plane can be scaled + // up immediately when the rolling update starts. + // +optional + MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"` } // KubeadmControlPlaneStatus defines the observed state of KubeadmControlPlane. diff --git a/controlplane/kubeadm/api/v1alpha4/kubeadm_control_plane_webhook.go b/controlplane/kubeadm/api/v1alpha4/kubeadm_control_plane_webhook.go index a1eb29c0a4c1..e3e77475d22f 100644 --- a/controlplane/kubeadm/api/v1alpha4/kubeadm_control_plane_webhook.go +++ b/controlplane/kubeadm/api/v1alpha4/kubeadm_control_plane_webhook.go @@ -27,6 +27,7 @@ import ( "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/validation/field" kubeadmv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1" "sigs.k8s.io/cluster-api/util/container" @@ -61,6 +62,25 @@ func (in *KubeadmControlPlane) Default() { if !strings.HasPrefix(in.Spec.Version, "v") { in.Spec.Version = "v" + in.Spec.Version } + + ios1 := intstr.FromInt(1) + + if in.Spec.RolloutStrategy == nil { + in.Spec.RolloutStrategy = &RolloutStrategy{} + } + + // Enforce RollingUpdate strategy and default MaxSurge if not set. + if in.Spec.RolloutStrategy != nil { + if len(in.Spec.RolloutStrategy.Type) == 0 { + in.Spec.RolloutStrategy.Type = RollingUpdateStrategyType + } + if in.Spec.RolloutStrategy.Type == RollingUpdateStrategyType { + if in.Spec.RolloutStrategy.RollingUpdate == nil { + in.Spec.RolloutStrategy.RollingUpdate = &RollingUpdate{} + } + in.Spec.RolloutStrategy.RollingUpdate.MaxSurge = intstr.ValueOrDefault(in.Spec.RolloutStrategy.RollingUpdate.MaxSurge, ios1) + } + } } // ValidateCreate implements webhook.Validator so a webhook will be registered for the type @@ -116,6 +136,7 @@ func (in *KubeadmControlPlane) ValidateUpdate(old runtime.Object) error { {spec, "version"}, {spec, "upgradeAfter"}, {spec, "nodeDrainTimeout"}, + {spec, "rolloutStrategy"}, } allErrs := in.validateCommon() @@ -269,6 +290,42 @@ func (in *KubeadmControlPlane) validateCommon() (allErrs field.ErrorList) { allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "version"), in.Spec.Version, "must be a valid semantic version")) } + if in.Spec.RolloutStrategy != nil { + + if in.Spec.RolloutStrategy.Type != RollingUpdateStrategyType { + allErrs = append( + allErrs, + field.Required( + field.NewPath("spec", "rolloutStrategy", "type"), + "only RollingUpdateStrategyType is supported", + ), + ) + } + + ios1 := intstr.FromInt(1) + ios0 := intstr.FromInt(0) + + if *in.Spec.RolloutStrategy.RollingUpdate.MaxSurge == ios0 && *in.Spec.Replicas < int32(3) { + allErrs = append( + allErrs, + field.Required( + field.NewPath("spec", "rolloutStrategy", "rollingUpdate"), + "when KubeadmControlPlane is configured to scale-in, replica count needs to be at least 3", + ), + ) + } + + if *in.Spec.RolloutStrategy.RollingUpdate.MaxSurge != ios1 && *in.Spec.RolloutStrategy.RollingUpdate.MaxSurge != ios0 { + allErrs = append( + allErrs, + field.Required( + field.NewPath("spec", "rolloutStrategy", "rollingUpdate", "maxSurge"), + "value must be 1 or 0", + ), + ) + } + } + allErrs = append(allErrs, in.validateCoreDNSImage()...) return allErrs diff --git a/controlplane/kubeadm/api/v1alpha4/kubeadm_control_plane_webhook_test.go b/controlplane/kubeadm/api/v1alpha4/kubeadm_control_plane_webhook_test.go index 0c349ab629c2..7d4c351c9c58 100644 --- a/controlplane/kubeadm/api/v1alpha4/kubeadm_control_plane_webhook_test.go +++ b/controlplane/kubeadm/api/v1alpha4/kubeadm_control_plane_webhook_test.go @@ -24,6 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha4" kubeadmv1beta1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1" @@ -37,14 +38,18 @@ func TestKubeadmControlPlaneDefault(t *testing.T) { Namespace: "foo", }, Spec: KubeadmControlPlaneSpec{ - InfrastructureTemplate: corev1.ObjectReference{}, Version: "1.18.3", + InfrastructureTemplate: corev1.ObjectReference{}, + RolloutStrategy: &RolloutStrategy{}, }, } kcp.Default() g.Expect(kcp.Spec.InfrastructureTemplate.Namespace).To(Equal(kcp.Namespace)) g.Expect(kcp.Spec.Version).To(Equal("v1.18.3")) + g.Expect(kcp.Spec.RolloutStrategy.Type).To(Equal(RollingUpdateStrategyType)) + g.Expect(kcp.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntVal).To(Equal(int32(1))) + } func TestKubeadmControlPlaneValidateCreate(t *testing.T) { @@ -60,8 +65,20 @@ func TestKubeadmControlPlaneValidateCreate(t *testing.T) { }, Replicas: pointer.Int32Ptr(1), Version: "v1.19.0", + RolloutStrategy: &RolloutStrategy{ + Type: RollingUpdateStrategyType, + RollingUpdate: &RollingUpdate{ + MaxSurge: &intstr.IntOrString{ + IntVal: 1, + }, + }, + }, }, } + + invalidMaxSurge := valid.DeepCopy() + invalidMaxSurge.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntVal = int32(3) + invalidNamespace := valid.DeepCopy() invalidNamespace.Spec.InfrastructureTemplate.Namespace = "bar" @@ -142,6 +159,11 @@ func TestKubeadmControlPlaneValidateCreate(t *testing.T) { expectErr: true, kcp: invalidVersion1, }, + { + name: "should return error when maxSurge is not 1", + expectErr: true, + kcp: invalidMaxSurge, + }, } for _, tt := range tests { diff --git a/controlplane/kubeadm/api/v1alpha4/zz_generated.deepcopy.go b/controlplane/kubeadm/api/v1alpha4/zz_generated.deepcopy.go index ee557c94cd9d..35406360f49a 100644 --- a/controlplane/kubeadm/api/v1alpha4/zz_generated.deepcopy.go +++ b/controlplane/kubeadm/api/v1alpha4/zz_generated.deepcopy.go @@ -23,6 +23,7 @@ package v1alpha4 import ( "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" apiv1alpha4 "sigs.k8s.io/cluster-api/api/v1alpha4" ) @@ -104,6 +105,11 @@ func (in *KubeadmControlPlaneSpec) DeepCopyInto(out *KubeadmControlPlaneSpec) { *out = new(v1.Duration) **out = **in } + if in.RolloutStrategy != nil { + in, out := &in.RolloutStrategy, &out.RolloutStrategy + *out = new(RolloutStrategy) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmControlPlaneSpec. @@ -142,3 +148,43 @@ func (in *KubeadmControlPlaneStatus) DeepCopy() *KubeadmControlPlaneStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RollingUpdate) DeepCopyInto(out *RollingUpdate) { + *out = *in + if in.MaxSurge != nil { + in, out := &in.MaxSurge, &out.MaxSurge + *out = new(intstr.IntOrString) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdate. +func (in *RollingUpdate) DeepCopy() *RollingUpdate { + if in == nil { + return nil + } + out := new(RollingUpdate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RolloutStrategy) DeepCopyInto(out *RolloutStrategy) { + *out = *in + if in.RollingUpdate != nil { + in, out := &in.RollingUpdate, &out.RollingUpdate + *out = new(RollingUpdate) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutStrategy. +func (in *RolloutStrategy) DeepCopy() *RolloutStrategy { + if in == nil { + return nil + } + out := new(RolloutStrategy) + in.DeepCopyInto(out) + return out +} diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml index 211418af3672..43acefa4335e 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml @@ -1517,6 +1517,23 @@ spec: description: Number of desired machines. Defaults to 1. When stacked etcd is used only odd numbers are permitted, as per [etcd best practice](https://etcd.io/docs/v3.3.12/faq/#why-an-odd-number-of-cluster-members). This is a pointer to distinguish between explicit zero and not specified. format: int32 type: integer + rolloutStrategy: + description: The RolloutStrategy to use to replace control plane machines with new ones. + properties: + rollingUpdate: + description: Rolling update config params. Present only if RolloutStrategyType = RollingUpdate. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: 'The maximum number of control planes that can be scheduled above or under the desired number of control planes. Value can be an absolute number 1 or 0. Defaults to 1. Example: when this is set to 1, the control plane can be scaled up immediately when the rolling update starts.' + x-kubernetes-int-or-string: true + type: object + type: + description: Type of rollout. Currently the only supported strategy is "RollingUpdate". Default is RollingUpdate. + type: string + type: object upgradeAfter: description: UpgradeAfter is a field to indicate an upgrade should be performed after the specified time even if no changes have been made to the KubeadmControlPlane format: date-time diff --git a/controlplane/kubeadm/controllers/controller_test.go b/controlplane/kubeadm/controllers/controller_test.go index 2f20ace550db..cc8e5e100f27 100644 --- a/controlplane/kubeadm/controllers/controller_test.go +++ b/controlplane/kubeadm/controllers/controller_test.go @@ -19,11 +19,12 @@ package controllers import ( "context" "fmt" - "sigs.k8s.io/cluster-api/util/collections" "sync" "testing" "time" + "sigs.k8s.io/cluster-api/util/collections" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -32,6 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" "k8s.io/klog/klogr" @@ -1287,6 +1289,14 @@ func createClusterWithControlPlane() (*clusterv1.Cluster, *controlplanev1.Kubead }, Replicas: pointer.Int32Ptr(int32(3)), Version: "v1.16.6", + RolloutStrategy: &controlplanev1.RolloutStrategy{ + Type: "RollingUpdate", + RollingUpdate: &controlplanev1.RollingUpdate{ + MaxSurge: &intstr.IntOrString{ + IntVal: 1, + }, + }, + }, }, } diff --git a/controlplane/kubeadm/controllers/remediation_test.go b/controlplane/kubeadm/controllers/remediation_test.go index 63ecd7bc5a68..8b4b287492f8 100644 --- a/controlplane/kubeadm/controllers/remediation_test.go +++ b/controlplane/kubeadm/controllers/remediation_test.go @@ -19,13 +19,15 @@ package controllers import ( "context" "fmt" - "sigs.k8s.io/cluster-api/util/collections" "testing" + "sigs.k8s.io/cluster-api/util/collections" + . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" utilpointer "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" @@ -82,6 +84,13 @@ func TestReconcileUnhealthyMachines(t *testing.T) { controlPlane := &internal.ControlPlane{ KCP: &controlplanev1.KubeadmControlPlane{Spec: controlplanev1.KubeadmControlPlaneSpec{ Replicas: utilpointer.Int32Ptr(1), + RolloutStrategy: &controlplanev1.RolloutStrategy{ + RollingUpdate: &controlplanev1.RollingUpdate{ + MaxSurge: &intstr.IntOrString{ + IntVal: 1, + }, + }, + }, }}, Cluster: &clusterv1.Cluster{}, Machines: collections.FromMachines(m), @@ -100,7 +109,8 @@ func TestReconcileUnhealthyMachines(t *testing.T) { m := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed()) controlPlane := &internal.ControlPlane{ KCP: &controlplanev1.KubeadmControlPlane{Spec: controlplanev1.KubeadmControlPlaneSpec{ - Replicas: utilpointer.Int32Ptr(3), + Replicas: utilpointer.Int32Ptr(3), + RolloutStrategy: &controlplanev1.RolloutStrategy{}, }}, Cluster: &clusterv1.Cluster{}, Machines: collections.FromMachines(m), diff --git a/controlplane/kubeadm/controllers/scale_test.go b/controlplane/kubeadm/controllers/scale_test.go index 394cb952d0ba..c3bb34b2d973 100644 --- a/controlplane/kubeadm/controllers/scale_test.go +++ b/controlplane/kubeadm/controllers/scale_test.go @@ -19,10 +19,11 @@ package controllers import ( "context" "fmt" - "sigs.k8s.io/cluster-api/util/collections" "testing" "time" + "sigs.k8s.io/cluster-api/util/collections" + . "github.com/onsi/gomega" "sigs.k8s.io/cluster-api/util/conditions" @@ -166,6 +167,7 @@ func TestKubeadmControlPlaneReconciler_scaleUpControlPlane(t *testing.T) { g.Expect(m).To(Equal(bm)) } }) + } func TestKubeadmControlPlaneReconciler_scaleDownControlPlane_NoError(t *testing.T) { diff --git a/controlplane/kubeadm/controllers/upgrade.go b/controlplane/kubeadm/controllers/upgrade.go index 910e5c80f27d..81d477de2f4b 100644 --- a/controlplane/kubeadm/controllers/upgrade.go +++ b/controlplane/kubeadm/controllers/upgrade.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "github.com/blang/semver" "github.com/pkg/errors" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" @@ -37,6 +38,10 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane( ) (ctrl.Result, error) { logger := controlPlane.Logger() + if kcp.Spec.RolloutStrategy == nil && kcp.Spec.RolloutStrategy.RollingUpdate == nil { + return ctrl.Result{}, errors.New("rolloutStrategy is not set") + } + // TODO: handle reconciliation of etcd members and kubeadm config in case they get out of sync with cluster workloadCluster, err := r.managementCluster.GetWorkloadCluster(ctx, util.ObjectKey(cluster)) @@ -105,9 +110,18 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane( return ctrl.Result{}, err } - if status.Nodes <= *kcp.Spec.Replicas { - // scaleUp ensures that we don't continue scaling up while waiting for Machines to have NodeRefs - return r.scaleUpControlPlane(ctx, cluster, kcp, controlPlane) + switch kcp.Spec.RolloutStrategy.Type { + case controlplanev1.RollingUpdateStrategyType: + // RolloutStrategy is currently defaulted and validated to be RollingUpdate + // We can ignore MaxUnavailable because we are enforcing health checks before we get here. + maxNodes := *kcp.Spec.Replicas + int32(kcp.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntValue()) + if status.Nodes < maxNodes { + // scaleUp ensures that we don't continue scaling up while waiting for Machines to have NodeRefs + return r.scaleUpControlPlane(ctx, cluster, kcp, controlPlane) + } + return r.scaleDownControlPlane(ctx, cluster, kcp, controlPlane, machinesRequireUpgrade) + default: + logger.Info("RolloutStrategy type is not set to RollingUpdateStrategyType, unable to determine the strategy for rolling out machines") + return ctrl.Result{}, nil } - return r.scaleDownControlPlane(ctx, cluster, kcp, controlPlane, machinesRequireUpgrade) } diff --git a/controlplane/kubeadm/controllers/upgrade_test.go b/controlplane/kubeadm/controllers/upgrade_test.go index b073e3529c94..a79c61b919aa 100644 --- a/controlplane/kubeadm/controllers/upgrade_test.go +++ b/controlplane/kubeadm/controllers/upgrade_test.go @@ -18,27 +18,34 @@ package controllers import ( "context" - "sigs.k8s.io/cluster-api/util/collections" + "fmt" "testing" + "sigs.k8s.io/cluster-api/util/collections" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha4" "sigs.k8s.io/cluster-api/controlplane/kubeadm/internal" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) -func TestKubeadmControlPlaneReconciler_upgradeControlPlane(t *testing.T) { +const UpdatedVersion string = "v1.17.4" +const Host string = "nodomain.example.com" + +func TestKubeadmControlPlaneReconciler_RolloutStrategy_ScaleUp(t *testing.T) { g := NewWithT(t) cluster, kcp, genericMachineTemplate := createClusterWithControlPlane() - cluster.Spec.ControlPlaneEndpoint.Host = "nodomain.example.com" + cluster.Spec.ControlPlaneEndpoint.Host = Host cluster.Spec.ControlPlaneEndpoint.Port = 6443 - kcp.Spec.Version = "v1.17.3" + //kcp.Spec.Version = Version kcp.Spec.KubeadmConfigSpec.ClusterConfiguration = nil kcp.Spec.Replicas = pointer.Int32Ptr(1) setKCPHealthy(kcp) @@ -80,7 +87,7 @@ func TestKubeadmControlPlaneReconciler_upgradeControlPlane(t *testing.T) { } // change the KCP spec so the machine becomes outdated - kcp.Spec.Version = "v1.17.4" + kcp.Spec.Version = UpdatedVersion // run upgrade the first time, expect we scale up needingUpgrade := collections.FromMachineList(initialMachine) @@ -121,6 +128,92 @@ func TestKubeadmControlPlaneReconciler_upgradeControlPlane(t *testing.T) { g.Expect(finalMachine.Items[0].CreationTimestamp.Time).To(BeTemporally(">", initialMachine.Items[0].CreationTimestamp.Time)) } +func TestKubeadmControlPlaneReconciler_RolloutStrategy_ScaleDown(t *testing.T) { + version := "v1.17.3" + g := NewWithT(t) + + cluster, kcp, tmpl := createClusterWithControlPlane() + cluster.Spec.ControlPlaneEndpoint.Host = "nodomain.example.com1" + cluster.Spec.ControlPlaneEndpoint.Port = 6443 + //kcp.Spec.Version = Version + kcp.Spec.Replicas = pointer.Int32Ptr(3) + kcp.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntVal = 0 + setKCPHealthy(kcp) + + fmc := &fakeManagementCluster{ + Machines: collections.Machines{}, + Workload: fakeWorkloadCluster{ + Status: internal.ClusterStatus{Nodes: 3}, + }, + } + objs := []client.Object{cluster.DeepCopy(), kcp.DeepCopy(), tmpl.DeepCopy()} + for i := 0; i < 3; i++ { + name := fmt.Sprintf("test-%d", i) + m := &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: name, + Labels: internal.ControlPlaneLabelsForCluster(cluster.Name), + }, + Spec: clusterv1.MachineSpec{ + Bootstrap: clusterv1.Bootstrap{ + ConfigRef: &corev1.ObjectReference{ + APIVersion: bootstrapv1.GroupVersion.String(), + Kind: "KubeadmConfig", + Name: name, + }, + }, + Version: &version, + }, + } + cfg := &bootstrapv1.KubeadmConfig{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: name, + }, + } + objs = append(objs, m, cfg) + fmc.Machines.Insert(m) + } + fakeClient := newFakeClient(g, objs...) + fmc.Reader = fakeClient + r := &KubeadmControlPlaneReconciler{ + Client: fakeClient, + managementCluster: fmc, + managementClusterUncached: fmc, + } + + controlPlane := &internal.ControlPlane{ + KCP: kcp, + Cluster: cluster, + Machines: nil, + } + + result, err := r.reconcile(ctx, cluster, kcp) + g.Expect(result).To(Equal(ctrl.Result{})) + g.Expect(err).NotTo(HaveOccurred()) + + machineList := &clusterv1.MachineList{} + g.Expect(fakeClient.List(ctx, machineList, client.InNamespace(cluster.Namespace))).To(Succeed()) + g.Expect(machineList.Items).To(HaveLen(3)) + for i := range machineList.Items { + setMachineHealthy(&machineList.Items[i]) + } + + // change the KCP spec so the machine becomes outdated + kcp.Spec.Version = UpdatedVersion + + // run upgrade, expect we scale down + needingUpgrade := collections.FromMachineList(machineList) + controlPlane.Machines = needingUpgrade + result, err = r.upgradeControlPlane(ctx, cluster, kcp, controlPlane, needingUpgrade) + g.Expect(result).To(Equal(ctrl.Result{Requeue: true})) + g.Expect(err).To(BeNil()) + remainingMachines := &clusterv1.MachineList{} + g.Expect(fakeClient.List(ctx, remainingMachines, client.InNamespace(cluster.Namespace))).To(Succeed()) + g.Expect(remainingMachines.Items).To(HaveLen(2)) +} + type machineOpt func(*clusterv1.Machine) func machine(name string, opts ...machineOpt) *clusterv1.Machine { diff --git a/docs/proposals/20191017-kubeadm-based-control-plane.md b/docs/proposals/20191017-kubeadm-based-control-plane.md index ba5ca00f8d6c..5eb23b5cf1fe 100644 --- a/docs/proposals/20191017-kubeadm-based-control-plane.md +++ b/docs/proposals/20191017-kubeadm-based-control-plane.md @@ -159,7 +159,7 @@ Non-Goals listed in this document are intended to scope bound the current v1alph 3. In service of user story 5, the kubeadm control plane provider must also manage etcd membership via kubeadm as part of scaling down (`kubeadm` takes care of adding the new etcd member when joining). 4. The control plane provider should provide indicators of health to meet user story 6 and 10. This should include at least the state of etcd and information about which replicas are currently healthy or not. For the default implementation, health attributes based on artifacts kubeadm installs on the cluster may also be of interest to cluster operators. 5. The control plane provider must be able to upgrade a control plane’s version of Kubernetes as well as updating the underlying machine image on where applicable (e.g. virtual machine based infrastructure). -6. To address user story 11, the control plane provider must provide Rolling Update strategy similar to MachineDeployment. With `MaxUnavailable` and `MaxSurge` fields user is able to delete old machine first during upgrade. Control plane provider should default the `RolloutStrategy`, `MaxUnavailable` and `MaxSurge` fields such a way that scaling up is the default behavior during upgrade. +6. To address user story 11, the control plane provider must provide Rolling Update strategy similar to MachineDeployment. With `MaxSurge` field user is able to delete old machine first during upgrade. Control plane provider should default the `RolloutStrategy` and `MaxSurge` fields such a way that scaling up is the default behavior during upgrade. ### Implementation Details/Notes/Constraints @@ -229,25 +229,13 @@ And the following defaulting: ```go // RollingUpdate is used to control the desired behavior of rolling update. type RollingUpdate struct { - // The maximum number of control planes that can be unavailable during the rollout. - // Value can be an absolute number 0 or 1. - // This needs to be 1 if MaxSurge is 0. - // Defaults to 0. - // Example: when this is set to 1 and MaxSurge is 0, the control planes can be scaled - // down one-by-one when the rolling update starts. - // Control plane scale down is disabled when desired number of control planes is 1. - // Scale down is possible only if desired number of control planes is 3 or more. - // +optional - MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` - - // The maximum number of control planes that can be scheduled above the - // desired number of control planes. - // Value can be an absolute number 1 or 0. - // This needs to be 1 if MaxUnavailable is 0. - // Defaults to 1. - // Example: when this is set to 1 and MaxUnavailable is 0, the control plane can be scaled - // up immediately when the rolling update starts. - // +optional + // The maximum number of control planes that can be scheduled above or under the + // desired number of control planes. + // Value can be an absolute number 1 or 0. + // Defaults to 1. + // Example: when this is set to 1, the control plane can be scaled + // up immediately when the rolling update starts. + // +optional MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"` } ``` @@ -448,25 +436,22 @@ KubeadmControlPlane rollout is triggered by: ##### Rolling update strategy -Currently KubeadmControlPlane supports only one rollout strategy type the `RollingUpdateStrategyType`. Rolling upgrade strategy's behavior can be modified by using `MaxUnavailable` and `MaxSurge` fields. Both field values can be an absolute number 0 or 1 with following rules: - -- If `MaxUnavailable` is set to 0 `MaxSurge` needs to be 1 (default values) -- If `MaxUnavailable` is set to 1 `MaxSurge` needs to be 0 +Currently KubeadmControlPlane supports only one rollout strategy type the `RollingUpdateStrategyType`. Rolling upgrade strategy's behavior can be modified by using `MaxSurge` field. The field values can be an absolute number 0 or 1. -When `MaxUnavailable` is set to 0 and `MaxSurge` is set to 1 the rollout algorithm is as follows: +When `MaxSurge` is set to 1 the rollout algorithm is as follows: - Find Machines that have an outdated spec - If there is a machine requiring rollout - Scale up control plane creating a machine with the new spec - Scale down control plane by removing one of the machine that needs rollout (the oldest out-of date machine in the failure domain that has the most control-plane machines on it) -When `MaxUnavailable` is set to 1 and `MaxSurge` is set to 0 the rollout algorithm is as follows: +When `MaxSurge` is set to 0 the rollout algorithm is as follows: - KubeadmControlPlane verifies that control plane replica count is >= 3 - Find Machines that have an outdated spec and scale down the control plane by removing the oldest out-of-date machine. - Scale up control plane by creating a new machine with the updated spec -> NOTE: Setting `MaxUnavailable` to 1 and `MaxSurge` to 0 could be use in resource constrained environment like bare-metal, OpenStack or vSphere resource pools, etc when there is no capacity to Scale up the control plane. +> NOTE: Setting `MaxSurge` to 0 could be use in resource constrained environment like bare-metal, OpenStack or vSphere resource pools, etc when there is no capacity to Scale up the control plane. ###### Constraints and Assumptions