Skip to content

OCPBUGS-54611: pkg/operator/status: Drop kubelet skew guard #4970

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-bot merged 1 commit into from
Apr 9, 2025
Merged

OCPBUGS-54611: pkg/operator/status: Drop kubelet skew guard #4970

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 1 addition & 160 deletions pkg/operator/status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"fmt"
"reflect"
"sort"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -255,13 +254,6 @@ func (optr *Operator) syncDegradedStatus(co *configv1.ClusterOperator, ierr sync
cov1helpers.SetStatusCondition(&co.Status.Conditions, coDegradedCondition, clock.RealClock{})
}

const (
skewUnchecked = "KubeletSkewUnchecked"
skewSupported = "KubeletSkewSupported"
skewUnsupported = "KubeletSkewUnsupported"
skewPresent = "KubeletSkewPresent"
)

// syncUpgradeableStatus applies the new condition to the mco's ClusterOperator object.
func (optr *Operator) syncUpgradeableStatus(co *configv1.ClusterOperator) error {

Expand Down Expand Up @@ -289,13 +281,8 @@ func (optr *Operator) syncUpgradeableStatus(co *configv1.ClusterOperator) error
coStatusCondition.Reason = "ClusterOnCgroupV1"
coStatusCondition.Message = "Cluster is using deprecated cgroup v1 and is not upgradable. Please update the `CgroupMode` in the `nodes.config.openshift.io` object to 'v2'. Once upgraded, the cluster cannot be changed back to cgroup v1"
}
var updating, degraded, interrupted bool
var degraded, interrupted bool
for _, pool := range pools {
// collect updating status but continue to check each pool to see if any pool is degraded
if isPoolStatusConditionTrue(pool, mcfgv1.MachineConfigPoolUpdating) {
updating = true
}

interrupted = isPoolStatusConditionTrue(pool, mcfgv1.MachineConfigPoolBuildInterrupted)

degraded = isPoolStatusConditionTrue(pool, mcfgv1.MachineConfigPoolDegraded)
Expand All @@ -315,39 +302,6 @@ func (optr *Operator) syncUpgradeableStatus(co *configv1.ClusterOperator) error
}
}

// don't overwrite status if updating or degraded
if !updating && !degraded && !interrupted {
skewStatus, status, err := optr.isKubeletSkewSupported(pools)
if err != nil {
klog.Errorf("Error checking version skew: %v, kubelet skew status: %v, status reason: %v, status message: %v", err, skewStatus, status.Reason, status.Message)
coStatusCondition.Reason = status.Reason
coStatusCondition.Message = status.Message
cov1helpers.SetStatusCondition(&co.Status.Conditions, coStatusCondition, clock.RealClock{})
}
switch skewStatus {
case skewUnchecked:
coStatusCondition.Reason = status.Reason
coStatusCondition.Message = status.Message
cov1helpers.SetStatusCondition(&co.Status.Conditions, coStatusCondition, clock.RealClock{})
case skewUnsupported:
coStatusCondition.Reason = status.Reason
coStatusCondition.Message = status.Message
mcoObjectRef := &corev1.ObjectReference{
Kind: co.Kind,
Name: co.Name,
Namespace: co.Namespace,
UID: co.GetUID(),
}
klog.Infof("kubelet skew status: %v, status reason: %v", skewStatus, status.Reason)
optr.eventRecorder.Eventf(mcoObjectRef, corev1.EventTypeWarning, coStatusCondition.Reason, coStatusCondition.Message)
cov1helpers.SetStatusCondition(&co.Status.Conditions, coStatusCondition, clock.RealClock{})
case skewPresent:
coStatusCondition.Reason = status.Reason
coStatusCondition.Message = status.Message
klog.Infof("kubelet skew status: %v, status reason: %v", skewStatus, status.Reason)
cov1helpers.SetStatusCondition(&co.Status.Conditions, coStatusCondition, clock.RealClock{})
}
}
cov1helpers.SetStatusCondition(&co.Status.Conditions, coStatusCondition, clock.RealClock{})
return nil
}
Expand Down Expand Up @@ -519,85 +473,6 @@ func (optr *Operator) cfeEvalCgroupsV1() (bool, error) {
return nodeClusterConfig.Spec.CgroupMode == configv1.CgroupModeV1, nil
}

// isKubeletSkewSupported checks the version skew of kube-apiserver and node kubelet version.
// Returns the skew status. version skew > 2 is not supported.
func (optr *Operator) isKubeletSkewSupported(pools []*mcfgv1.MachineConfigPool) (skewStatus string, coStatus configv1.ClusterOperatorStatusCondition, err error) {
coStatus = configv1.ClusterOperatorStatusCondition{}
kubeAPIServerStatus, err := optr.clusterOperatorLister.Get("kube-apiserver")
if err != nil {
coStatus.Reason = skewUnchecked
coStatus.Message = fmt.Sprintf("An error occurred when checking kubelet version skew: %v", err)
return skewUnchecked, coStatus, err
}
// looks like
// - name: kube-apiserver
// version: 1.21.0-rc.0
kubeAPIServerVersion := ""
for _, version := range kubeAPIServerStatus.Status.Versions {
if version.Name != "kube-apiserver" {
continue
}
kubeAPIServerVersion = version.Version
break
}
if kubeAPIServerVersion == "" {
err = fmt.Errorf("kube-apiserver does not yet have a version")
coStatus.Reason = skewUnchecked
coStatus.Message = fmt.Sprintf("An error occurred when checking kubelet version skew: %v", err.Error())
return skewUnchecked, coStatus, err
}
kubeAPIServerMinorVersion, err := getMinorKubeletVersion(kubeAPIServerVersion)
if err != nil {
coStatus.Reason = skewUnchecked
coStatus.Message = fmt.Sprintf("An error occurred when checking kubelet version skew: %v", err)
return skewUnchecked, coStatus, err
}
var (
lastError error
kubeletVersion string
)
nodes, err := optr.GetAllManagedNodes(pools)
if err != nil {
err = fmt.Errorf("getting all managed nodes failed: %w", err)
coStatus.Reason = skewUnchecked
coStatus.Message = fmt.Sprintf("An error occurred when getting all the managed nodes: %v", err.Error())
}
for _, node := range nodes {
// looks like kubeletVersion: v1.21.0-rc.0+6143dea
kubeletVersion = node.Status.NodeInfo.KubeletVersion
if kubeletVersion == "" {
continue
}
nodeMinorVersion, err := getMinorKubeletVersion(kubeletVersion)
if err != nil {
lastError = err
continue
}
if nodeMinorVersion+2 < kubeAPIServerMinorVersion {
coStatus.Reason = skewUnsupported
coStatus.Message = fmt.Sprintf("One or more nodes have an unsupported kubelet version skew. Please see `oc get nodes` for details and upgrade all nodes so that they have a kubelet version of at least %v.", getMinimalSkewSupportNodeVersion(kubeAPIServerVersion))
return skewUnsupported, coStatus, nil
}
if nodeMinorVersion+2 == kubeAPIServerMinorVersion {
coStatus.Reason = skewPresent
coStatus.Message = fmt.Sprintf("Current kubelet version %v will not be supported by newer kube-apiserver. Please upgrade the kubelet first if plan to upgrade the kube-apiserver", kubeletVersion)
return skewPresent, coStatus, nil
}
}
if kubeletVersion == "" {
err = fmt.Errorf("kubelet does not yet have a version")
coStatus.Reason = skewUnchecked
coStatus.Message = fmt.Sprintf("An error occurred when checking kubelet version skew: %v", err.Error())
return skewUnchecked, coStatus, err
}
if lastError != nil {
coStatus.Reason = skewUnchecked
coStatus.Message = fmt.Sprintf("An error occurred when checking kubelet version skew: %v", err)
return skewUnchecked, coStatus, lastError
}
return skewSupported, coStatus, nil
}

// GetAllManagedNodes returns the nodes managed by MCO
func (optr *Operator) GetAllManagedNodes(pools []*mcfgv1.MachineConfigPool) ([]*corev1.Node, error) {
nodes := []*corev1.Node{}
Expand All @@ -615,40 +490,6 @@ func (optr *Operator) GetAllManagedNodes(pools []*mcfgv1.MachineConfigPool) ([]*
return nodes, nil
}

// getMinorKubeletVersion parses the minor version number of kubelet
func getMinorKubeletVersion(version string) (int, error) {
tokens := strings.Split(version, ".")
if len(tokens) < 2 {
return 0, fmt.Errorf("incorrect version syntax: %q", version)
}
minorVersion, err := strconv.ParseInt(tokens[1], 10, 32)
if err != nil {
return 0, err
}
return int(minorVersion), nil
}

// getMinimalSkewSupportNodeVersion returns the minimal supported node kubelet version.
func getMinimalSkewSupportNodeVersion(version string) string {
// drop the pre-release and commit hash
idx := strings.Index(version, "-")
if idx >= 0 {
version = version[:idx]
}

idx = strings.Index(version, "+")
if idx >= 0 {
version = version[:idx]
}

tokens := strings.Split(version, ".")
if minorVersion, err := strconv.ParseInt(tokens[1], 10, 32); err == nil {
tokens[1] = strconv.Itoa(int(minorVersion) - 2)
return strings.Join(tokens, ".")
}
return version
}

func (optr *Operator) fetchClusterOperator() (*configv1.ClusterOperator, error) {
co, err := optr.clusterOperatorLister.Get(optr.name)

Expand Down
Loading