[OCPNODE-852] workerlatency profiles - updating the kubelet configuration based on the nodes.config.openshift.io resource

cmd/machine-config-controller/start.go

@@ -116,9 +116,11 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle
 			ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(),
 			ctx.InformerFactory.Machineconfiguration().V1().KubeletConfigs(),
 			ctx.ConfigInformerFactory.Config().V1().FeatureGates(),
+			ctx.ConfigInformerFactory.Config().V1().Nodes(),
 			ctx.ConfigInformerFactory.Config().V1().APIServers(),
 			ctx.ClientBuilder.KubeClientOrDie("kubelet-config-controller"),
 			ctx.ClientBuilder.MachineConfigClientOrDie("kubelet-config-controller"),
+			ctx.ClientBuilder.ConfigClientOrDie("kubelet-config-controller"),
 		),
 		containerruntimeconfig.New(
 			rootOpts.templates,

go.mod

@@ -25,8 +25,8 @@ require (
 	github.com/google/renameio v0.1.0
 	github.com/imdario/mergo v0.3.12
 	github.com/opencontainers/go-digest v1.0.0
-	github.com/openshift/api v0.0.0-20220310165943-abf6417c2748
-	github.com/openshift/client-go v0.0.0-20211209144617-7385dd6338e3
+	github.com/openshift/api v0.0.0-20220412161459-8f38b7648620
+	github.com/openshift/client-go v0.0.0-20220411091747-c100636b0bc0
 	github.com/openshift/library-go v0.0.0-20211220195323-eca2c467c492
 	github.com/openshift/runtime-utils v0.0.0-20220225175100-8dec0d84fb39
 	github.com/pkg/errors v0.9.1

go.sum

@@ -1057,13 +1057,15 @@ github.com/opencontainers/selinux v1.8.5/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4
 github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/openshift/api v0.0.0-20200326160804-ecb9283fe820/go.mod h1:RKMJ5CBnljLfnej+BJ/xnOWc3kZDvJUaIAEq2oKSPtE=
 github.com/openshift/api v0.0.0-20211209135129-c58d9f695577/go.mod h1:DoslCwtqUpr3d/gsbq4ZlkaMEdYqKxuypsDjorcHhME=
-github.com/openshift/api v0.0.0-20220310165943-abf6417c2748 h1:+LW/1gfPO3IXxgJKfdwgVAc7fdTj+rB1I8rSVGHJVFU=
-github.com/openshift/api v0.0.0-20220310165943-abf6417c2748/go.mod h1:F/eU6jgr6Q2VhMu1mSpMmygxAELd7+BUxs3NHZ25jV4=
+github.com/openshift/api v0.0.0-20220409151926-43bbec6227d5/go.mod h1:F/eU6jgr6Q2VhMu1mSpMmygxAELd7+BUxs3NHZ25jV4=
+github.com/openshift/api v0.0.0-20220412161459-8f38b7648620 h1:eNClZcEJo3EUIMCkK2GPrXl4lrFNfyKGRFVrUEZ8JIk=
+github.com/openshift/api v0.0.0-20220412161459-8f38b7648620/go.mod h1:F/eU6jgr6Q2VhMu1mSpMmygxAELd7+BUxs3NHZ25jV4=
 github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc=
 github.com/openshift/build-machinery-go v0.0.0-20210712174854-1bb7fd1518d3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
 github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
-github.com/openshift/client-go v0.0.0-20211209144617-7385dd6338e3 h1:SG1aqwleU6bGD0X4mhkTNupjVnByMYYuW4XbnCPavQU=
 github.com/openshift/client-go v0.0.0-20211209144617-7385dd6338e3/go.mod h1:cwhyki5lqBmrT0m8Im+9I7PGFaraOzcYPtEz93RcsGY=
+github.com/openshift/client-go v0.0.0-20220411091747-c100636b0bc0 h1:dZZJT2aINho70JcEDs85fW0E+Ug9aVmeDDHdvJYnroA=
+github.com/openshift/client-go v0.0.0-20220411091747-c100636b0bc0/go.mod h1:dUE4zBPh9kKaPm9e8qbMBh5RDu3zEO2jo0md4veGYaI=
 github.com/openshift/library-go v0.0.0-20211220195323-eca2c467c492 h1:oj/rSQqVWVj6YJUydZwLz2frrJreiyI4oa9g/YPgMsM=
 github.com/openshift/library-go v0.0.0-20211220195323-eca2c467c492/go.mod h1:4UQ9snU1vg53fyTpHQw3vLPiAxI8ub5xrc+y8KPQQFs=
 github.com/openshift/runtime-utils v0.0.0-20220225175100-8dec0d84fb39 h1:5woAtivjIZDVaRdMRDstmSBd0hHLsdYcdumLCMs0dwk=

manifests/machineconfigcontroller/clusterrole.yaml

@@ -13,7 +13,7 @@ rules:
   resources: ["configmaps", "secrets"]
   verbs: ["*"]
 - apiGroups: ["config.openshift.io"]
-  resources: ["images", "clusterversions", "featuregates"]
+  resources: ["images", "clusterversions", "featuregates", "nodes", "nodes/status"]
   verbs: ["*"]
 - apiGroups: ["config.openshift.io"]
   resources: ["schedulers", "apiservers", "infrastructures"]

pkg/apis/machineconfiguration.openshift.io/v1/types.go

@@ -13,6 +13,9 @@ import (
 // For example: `master` or `worker`
 const MachineConfigRoleLabelKey = "machineconfiguration.openshift.io/role"
 
+// KubeletConfigRoleLabelPrefix is the label that must be present in the KubeletConfig CR
+const KubeletConfigRoleLabelPrefix = "pools.operator.machineconfiguration.openshift.io/"
+
 // +genclient
 // +genclient:nonNamespaced
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/controller/bootstrap/bootstrap.go

@@ -74,6 +74,7 @@ func (b *Bootstrap) Run(destDir string) error {
 
 	var cconfig *mcfgv1.ControllerConfig
 	var featureGate *apicfgv1.FeatureGate
+	var nodeConfig *apicfgv1.Node
 	var kconfigs []*mcfgv1.KubeletConfig
 	var pools []*mcfgv1.MachineConfigPool
 	var configs []*mcfgv1.MachineConfig
@@ -126,6 +127,10 @@ func (b *Bootstrap) Run(destDir string) error {
 				if obj.GetName() == ctrlcommon.ClusterFeatureInstanceName {
 					featureGate = obj
 				}
+			case *apicfgv1.Node:
+				if obj.GetName() == ctrlcommon.ClusterNodeInstanceName {
+					nodeConfig = obj
+				}
 			default:
 				glog.Infof("skipping %q [%d] manifest because of unhandled %T", file.Name(), idx+1, obji)
 			}
@@ -156,14 +161,22 @@ func (b *Bootstrap) Run(destDir string) error {
 		configs = append(configs, containerRuntimeConfigs...)
 	}
 	if featureGate != nil {
-		featureConfigs, err := kubeletconfig.RunFeatureGateBootstrap(b.templatesDir, featureGate, cconfig, pools)
+		featureConfigs, err := kubeletconfig.RunFeatureGateBootstrap(b.templatesDir, featureGate, nodeConfig, cconfig, pools)
 		if err != nil {
 			return err
 		}
 		configs = append(configs, featureConfigs...)
 	}
+
+	if nodeConfig != nil {
+		nodeConfigs, err := kubeletconfig.RunNodeConfigBootstrap(b.templatesDir, featureGate, cconfig, nodeConfig, pools)
+		if err != nil {
+			return err
+		}
+		configs = append(configs, nodeConfigs...)
+	}
 	if len(kconfigs) > 0 {
-		kconfigs, err := kubeletconfig.RunKubeletBootstrap(b.templatesDir, kconfigs, cconfig, featureGate, pools)
+		kconfigs, err := kubeletconfig.RunKubeletBootstrap(b.templatesDir, kconfigs, cconfig, featureGate, nodeConfig, pools)
 		if err != nil {
 			return err
 		}

pkg/controller/common/constants.go

@@ -30,4 +30,12 @@ const (
 
 	// ClusterFeatureInstanceName is a singleton name for featureGate configuration
 	ClusterFeatureInstanceName = "cluster"
+
+	// ClusterNodeInstanceName is a singleton name for node configuration
+	ClusterNodeInstanceName = "cluster"
+
+	// MachineConfigPoolMaster is the MachineConfigPool name given to the master
+	MachineConfigPoolMaster = "master"
+	// MachineConfigPoolWorker is the MachineConfigPool name given to the worker
+	MachineConfigPoolWorker = "worker"
 )

pkg/controller/kubelet-config/helpers.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"reflect"
 	"strconv"
 
 	ign3types "github.com/coreos/ignition/v2/config/v3_2/types"
@@ -131,6 +132,42 @@ func createNewDefaultFeatureGate() *osev1.FeatureGate {
 	}
 }
 
+func createNewDefaultNodeconfig() *osev1.Node {
+	return &osev1.Node{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: ctrlcommon.ClusterNodeInstanceName,
+		},
+		Spec: osev1.NodeSpec{
+			CgroupMode:           osev1.CgroupModeDefault,
+			WorkerLatencyProfile: osev1.DefaultUpdateDefaultReaction,
+		},
+	}
+}
+
+// updateOriginalKubeConfigwithNodeConfig updates the original Kubelet Configuration based on the Nodespecific configuration
+func updateOriginalKubeConfigwithNodeConfig(node *osev1.Node, originalKubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) error {
+	if node == nil {
+		return fmt.Errorf("node configuration not found, failed to update the original kubelet configuration")
+	}
+	if reflect.DeepEqual(node.Spec, osev1.NodeSpec{}) {
+		return fmt.Errorf("empty node resource spec found")
+	}
+	// updating the kubelet specific fields based on the Node's workerlatency profile.
+	// (TODO): The durations can be replaced with the defined constants in the openshift/api repository once the respective changes are merged.
+	switch node.Spec.WorkerLatencyProfile {
+	case osev1.MediumUpdateAverageReaction:
+		originalKubeletConfig.NodeStatusUpdateFrequency = metav1.Duration{Duration: osev1.MediumNodeStatusUpdateFrequency}
+	case osev1.LowUpdateSlowReaction:
+		originalKubeletConfig.NodeStatusUpdateFrequency = metav1.Duration{Duration: osev1.LowNodeStatusUpdateFrequency}
+	case osev1.DefaultUpdateDefaultReaction:
+		originalKubeletConfig.NodeStatusUpdateFrequency = metav1.Duration{Duration: osev1.DefaultNodeStatusUpdateFrequency}
+	default:
+		return fmt.Errorf("unknown worker latency profile type found %v, failed to update the original kubelet configuration", node.Spec.WorkerLatencyProfile)
+	}
+	// The kubelet configuration can be updated based on the cgroupmode as well here.
+	return nil
+}
+
 func findKubeletConfig(mc *mcfgv1.MachineConfig) (*ign3types.File, error) {
 	ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw)
 	if err != nil {
@@ -227,6 +264,10 @@ func getManagedFeaturesKeyDeprecated(pool *mcfgv1.MachineConfigPool) string {
 	return fmt.Sprintf("98-%s-%s-kubelet", pool.Name, pool.ObjectMeta.UID)
 }
 
+func getManagedNodeConfigKey(pool *mcfgv1.MachineConfigPool, client mcfgclientset.Interface) (string, error) {
+	return ctrlcommon.GetManagedKey(pool, client, "97", "kubelet", fmt.Sprintf("97-%s-%s-kubelet", pool.Name, pool.ObjectMeta.UID))
+}
+
 // Deprecated: use getManagedKubeletConfigKey
 func getManagedKubeletConfigKeyDeprecated(pool *mcfgv1.MachineConfigPool) string {
 	return fmt.Sprintf("99-%s-%s-kubelet", pool.Name, pool.ObjectMeta.UID)

pkg/controller/kubelet-config/kubelet_config_bootstrap.go

@@ -13,7 +13,7 @@ import (
 )
 
 // RunKubeletBootstrap generates MachineConfig objects for mcpPools that would have been generated by syncKubeletConfig
-func RunKubeletBootstrap(templateDir string, kubeletConfigs []*mcfgv1.KubeletConfig, controllerConfig *mcfgv1.ControllerConfig, features *configv1.FeatureGate, mcpPools []*mcfgv1.MachineConfigPool) ([]*mcfgv1.MachineConfig, error) {
+func RunKubeletBootstrap(templateDir string, kubeletConfigs []*mcfgv1.KubeletConfig, controllerConfig *mcfgv1.ControllerConfig, features *configv1.FeatureGate, nodeConfig *configv1.Node, mcpPools []*mcfgv1.MachineConfigPool) ([]*mcfgv1.MachineConfig, error) {
 	var res []*mcfgv1.MachineConfig
 	managedKeyExist := make(map[string]bool)
 	userDefinedSystemReserved := make(map[string]string)
@@ -43,6 +43,13 @@ func RunKubeletBootstrap(templateDir string, kubeletConfigs []*mcfgv1.KubeletCon
 			if err != nil {
 				return nil, err
 			}
+			// updating the originalKubeConfig based on the nodeConfig on a worker node
+			if role == ctrlcommon.MachineConfigPoolWorker {
+				if nodeConfig == nil {
+					nodeConfig = createNewDefaultNodeconfig()
+				}
+				updateOriginalKubeConfigwithNodeConfig(nodeConfig, originalKubeConfig)
+			}
 			if kubeletConfig.Spec.TLSSecurityProfile != nil {
 				// Inject TLS Options from Spec
 				observedMinTLSVersion, observedCipherSuites := getSecurityProfileCiphers(kubeletConfig.Spec.TLSSecurityProfile)

pkg/controller/kubelet-config/kubelet_config_bootstrap_test.go

@@ -74,7 +74,7 @@ func TestRunKubeletBootstrap(t *testing.T) {
 				},
 			}
 
-			mcs, err := RunKubeletBootstrap("../../../templates", cfgs, cc, nil, pools)
+			mcs, err := RunKubeletBootstrap("../../../templates", cfgs, cc, nil, nil, pools)
 			require.NoError(t, err)
 			require.Len(t, mcs, len(cfgs))
 

pkg/controller/kubelet-config/kubelet_config_controller.go

@@ -31,6 +31,7 @@ import (
 	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
 
 	configv1 "github.com/openshift/api/config/v1"
+	configclientset "github.com/openshift/client-go/config/clientset/versioned"
 	oseinformersv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1"
 	oselistersv1 "github.com/openshift/client-go/config/listers/config/v1"
 	"github.com/openshift/library-go/pkg/crypto"
@@ -75,6 +76,7 @@ type Controller struct {
 	templatesDir string
 
 	client        mcfgclientset.Interface
+	configClient  configclientset.Interface
 	eventRecorder record.EventRecorder
 
 	syncHandler          func(mcp string) error
@@ -92,11 +94,15 @@ type Controller struct {
 	featLister       oselistersv1.FeatureGateLister
 	featListerSynced cache.InformerSynced
 
+	nodeConfigLister       oselistersv1.NodeLister
+	nodeConfigListerSynced cache.InformerSynced
+
 	apiserverLister       oselistersv1.APIServerLister
 	apiserverListerSynced cache.InformerSynced
 
-	queue        workqueue.RateLimitingInterface
-	featureQueue workqueue.RateLimitingInterface
+	queue           workqueue.RateLimitingInterface
+	featureQueue    workqueue.RateLimitingInterface
+	nodeConfigQueue workqueue.RateLimitingInterface
 }
 
 // New returns a new kubelet config controller
@@ -106,20 +112,24 @@ func New(
 	ccInformer mcfginformersv1.ControllerConfigInformer,
 	mkuInformer mcfginformersv1.KubeletConfigInformer,
 	featInformer oseinformersv1.FeatureGateInformer,
+	nodeConfigInformer oseinformersv1.NodeInformer,
 	apiserverInformer oseinformersv1.APIServerInformer,
 	kubeClient clientset.Interface,
 	mcfgClient mcfgclientset.Interface,
+	configclient configclientset.Interface,
 ) *Controller {
 	eventBroadcaster := record.NewBroadcaster()
 	eventBroadcaster.StartLogging(glog.Infof)
 	eventBroadcaster.StartRecordingToSink(&coreclientsetv1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")})
 
 	ctrl := &Controller{
-		templatesDir:  templatesDir,
-		client:        mcfgClient,
-		eventRecorder: eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "machineconfigcontroller-kubeletconfigcontroller"}),
-		queue:         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machineconfigcontroller-kubeletconfigcontroller"),
-		featureQueue:  workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machineconfigcontroller-featurecontroller"),
+		templatesDir:    templatesDir,
+		client:          mcfgClient,
+		configClient:    configclient,
+		eventRecorder:   eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "machineconfigcontroller-kubeletconfigcontroller"}),
+		queue:           workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machineconfigcontroller-kubeletconfigcontroller"),
+		featureQueue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machineconfigcontroller-featurecontroller"),
+		nodeConfigQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machineconfigcontroller-nodeConfigcontroller"),
 	}
 
 	mkuInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
@@ -134,6 +144,12 @@ func New(
 		DeleteFunc: ctrl.deleteFeature,
 	})
 
+	nodeConfigInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    ctrl.addNodeConfig,
+		UpdateFunc: ctrl.updateNodeConfig,
+		DeleteFunc: ctrl.deleteNodeConfig,
+	})
+
 	ctrl.syncHandler = ctrl.syncKubeletConfig
 	ctrl.enqueueKubeletConfig = ctrl.enqueue
 
@@ -149,6 +165,9 @@ func New(
 	ctrl.featLister = featInformer.Lister()
 	ctrl.featListerSynced = featInformer.Informer().HasSynced
 
+	ctrl.nodeConfigLister = nodeConfigInformer.Lister()
+	ctrl.nodeConfigListerSynced = nodeConfigInformer.Informer().HasSynced
+
 	ctrl.apiserverLister = apiserverInformer.Lister()
 	ctrl.apiserverListerSynced = apiserverInformer.Informer().HasSynced
 
@@ -160,6 +179,7 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) {
 	defer utilruntime.HandleCrash()
 	defer ctrl.queue.ShutDown()
 	defer ctrl.featureQueue.ShutDown()
+	defer ctrl.nodeConfigQueue.ShutDown()
 
 	if !cache.WaitForCacheSync(stopCh, ctrl.mcpListerSynced, ctrl.mckListerSynced, ctrl.ccListerSynced, ctrl.featListerSynced, ctrl.apiserverListerSynced) {
 		return
@@ -176,6 +196,10 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) {
 		go wait.Until(ctrl.featureWorker, time.Second, stopCh)
 	}
 
+	for i := 0; i < workers; i++ {
+		go wait.Until(ctrl.nodeConfigWorker, time.Second, stopCh)
+	}
+
 	<-stopCh
 }
 
@@ -507,6 +531,15 @@ func (ctrl *Controller) syncKubeletConfig(key string) error {
 		return ctrl.syncStatusOnly(cfg, err)
 	}
 
+	// Fetch the NodeConfig
+	nodeConfig, err := ctrl.nodeConfigLister.Get(ctrlcommon.ClusterNodeInstanceName)
+	if macherrors.IsNotFound(err) {
+		nodeConfig = createNewDefaultNodeconfig()
+	} else if err != nil {
+		err := fmt.Errorf("could not fetch Node: %v", err)
+		return ctrl.syncStatusOnly(cfg, err)
+	}
+
 	for _, pool := range mcpPools {
 		if pool.Spec.Configuration.Name == "" {
 			updateDelay := 5 * time.Second
@@ -540,6 +573,10 @@ func (ctrl *Controller) syncKubeletConfig(key string) error {
 		if err != nil {
 			return ctrl.syncStatusOnly(cfg, err, "could not get original kubelet config: %v", err)
 		}
+		// updating the originalKubeConfig based on the nodeConfig on a worker node
+		if role == ctrlcommon.MachineConfigPoolWorker {
+			updateOriginalKubeConfigwithNodeConfig(nodeConfig, originalKubeConfig)
+		}
 
 		// Get the default API Server Security Profile
 		var profile *configv1.TLSSecurityProfile