monitoring of a new resource - node.config.openshift.io

cmd/machine-config-operator/start.go

@@ -67,6 +67,7 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 			ctrlctx.KubeNamespacedInformerFactory.Core().V1().ConfigMaps(),
 			ctrlctx.KubeInformerFactory.Core().V1().ConfigMaps(),
 			ctrlctx.ConfigInformerFactory.Config().V1().Infrastructures(),
+			ctrlctx.ConfigInformerFactory.Config().V1().Nodes(),
 			ctrlctx.ConfigInformerFactory.Config().V1().Networks(),
 			ctrlctx.ConfigInformerFactory.Config().V1().Proxies(),
 			ctrlctx.ConfigInformerFactory.Config().V1().DNSes(),

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/imdario/mergo v0.3.12
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/openshift/api v0.0.0-20220310165943-abf6417c2748
-	github.com/openshift/client-go v0.0.0-20211209144617-7385dd6338e3
+	github.com/openshift/client-go v0.0.0-20220302123837-25b55b99bd24
 	github.com/openshift/library-go v0.0.0-20211220195323-eca2c467c492
 	github.com/openshift/runtime-utils v0.0.0-20220225175100-8dec0d84fb39
 	github.com/pkg/errors v0.9.1

go.sum

@@ -1057,13 +1057,15 @@ github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
 github.com/openshift/api v0.0.0-20200326160804-ecb9283fe820/go.mod h1:RKMJ5CBnljLfnej+BJ/xnOWc3kZDvJUaIAEq2oKSPtE=
 github.com/openshift/api v0.0.0-20211209135129-c58d9f695577/go.mod h1:DoslCwtqUpr3d/gsbq4ZlkaMEdYqKxuypsDjorcHhME=
+github.com/openshift/api v0.0.0-20220302114424-24043075985b/go.mod h1:F/eU6jgr6Q2VhMu1mSpMmygxAELd7+BUxs3NHZ25jV4=
 github.com/openshift/api v0.0.0-20220310165943-abf6417c2748 h1:+LW/1gfPO3IXxgJKfdwgVAc7fdTj+rB1I8rSVGHJVFU=
 github.com/openshift/api v0.0.0-20220310165943-abf6417c2748/go.mod h1:F/eU6jgr6Q2VhMu1mSpMmygxAELd7+BUxs3NHZ25jV4=
 github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc=
 github.com/openshift/build-machinery-go v0.0.0-20210712174854-1bb7fd1518d3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
 github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
-github.com/openshift/client-go v0.0.0-20211209144617-7385dd6338e3 h1:SG1aqwleU6bGD0X4mhkTNupjVnByMYYuW4XbnCPavQU=
 github.com/openshift/client-go v0.0.0-20211209144617-7385dd6338e3/go.mod h1:cwhyki5lqBmrT0m8Im+9I7PGFaraOzcYPtEz93RcsGY=
+github.com/openshift/client-go v0.0.0-20220302123837-25b55b99bd24 h1:EvNHCp29kVyvrdUIriIrk57Ss/aqGEzugRISEDTUnTc=
+github.com/openshift/client-go v0.0.0-20220302123837-25b55b99bd24/go.mod h1:77ko09OuApMeawCvY3lBJrbClGhnAtRjltEnMNwN0d4=
 github.com/openshift/library-go v0.0.0-20211220195323-eca2c467c492 h1:oj/rSQqVWVj6YJUydZwLz2frrJreiyI4oa9g/YPgMsM=
 github.com/openshift/library-go v0.0.0-20211220195323-eca2c467c492/go.mod h1:4UQ9snU1vg53fyTpHQw3vLPiAxI8ub5xrc+y8KPQQFs=
 github.com/openshift/runtime-utils v0.0.0-20220225175100-8dec0d84fb39 h1:5woAtivjIZDVaRdMRDstmSBd0hHLsdYcdumLCMs0dwk=

manifests/controllerconfig.crd.yaml

@@ -820,6 +820,108 @@ spec:
                 required:
                 - spec
                 type: object
+              node:
+                description: >-
+                  Node holds cluster-wide information about node specific
+                  features.
+                   Compatibility level 1: Stable within a major release for a minimum of 12 months or 3 minor releases (whichever is longer).
+                nullable: true
+                properties:
+                  apiVersion:
+                    description: >-
+                      APIVersion defines the versioned schema of this
+                      representation of an object. Servers should convert
+                      recognized schemas to the latest internal value, and may
+                      reject unrecognized values. More info:
+                      https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+                    type: string
+                  kind:
+                    description: >-
+                      Kind is a string value representing the REST resource
+                      this object represents. Servers may infer this from the
+                      endpoint the client submits requests to. Cannot be
+                      updated. In CamelCase. More info:
+                      https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+                    type: string
+                  metadata:
+                    type: object
+                  spec:
+                    description: spec holds user settable values for configuration
+                    type: object
+                    properties:
+                      cgroupMode:
+                        description: >-
+                          CgroupMode determines the cgroups version on the
+                          node
+                        type: string
+                      workerLatencyProfile:
+                        description: >-
+                          WorkerLatencyProfile determins the how fast the
+                          kubelet is updating the status and corresponding
+                          reaction of the cluster
+                        type: string
+                  status:
+                    description: status holds observed values.
+                    type: object
+                    properties:
+                      workerLatencyProfileStatus:
+                        description: >-
+                          WorkerLatencyProfileStatus provides the current
+                          status of WorkerLatencyProfile
+                        type: object
+                        properties:
+                          conditions:
+                            description: >-
+                              conditions describes the state of the
+                              WorkerLatencyProfile and related components
+                              (Kubelet or Controller Manager or Kube API
+                              Server)
+                            type: array
+                            items:
+                              type: object
+                              required:
+                                - lastTransitionTime
+                                - owner
+                                - status
+                                - type
+                              properties:
+                                lastTransitionTime:
+                                  description: >-
+                                    lastTransitionTime is the time of the last
+                                    update to the current status property.
+                                  type: string
+                                  format: date-time
+                                message:
+                                  description: >-
+                                    message provides additional information
+                                    about the current condition. This is only
+                                    to be consumed by humans.  It may contain
+                                    Line Feed characters (U+000A), which
+                                    should be rendered as new lines.
+                                  type: string
+                                owner:
+                                  description: >-
+                                    Owner specifies the operator that is
+                                    updating this condition
+                                  type: string
+                                reason:
+                                  description: >-
+                                    reason is the CamelCase reason for the
+                                    condition's current status.
+                                  type: string
+                                status:
+                                  description: >-
+                                    status of the condition, one of True,
+                                    False, Unknown.
+                                  type: string
+                                type:
+                                  description: >-
+                                    type specifies the aspect reported by this
+                                    condition.
+                                  type: string
+                required:
+                  - spec
+                type: object
               ipFamilies:
                 description: ipFamilies indicates the IP families in use by the cluster
                   network
@@ -958,6 +1060,7 @@ spec:
             - dns
             - images
             - infra
+            - node
             - ipFamilies
             - kubeAPIServerServingCAData
             - osImageURL

manifests/machineconfigcontroller/clusterrole.yaml

@@ -14,7 +14,7 @@ rules:
   resources: ["configmaps", "secrets"]
   verbs: ["*"]
 - apiGroups: ["config.openshift.io"]
-  resources: ["images", "clusterversions", "featuregates"]
+  resources: ["images", "clusterversions", "featuregates", "nodes"]
   verbs: ["*"]
 - apiGroups: ["config.openshift.io"]
   resources: ["schedulers", "apiservers", "infrastructures"]

pkg/apis/machineconfiguration.openshift.io/v1/types.go

@@ -84,6 +84,10 @@ type ControllerConfigSpec struct {
 	// +nullable
 	Infra *configv1.Infrastructure `json:"infra"`
 
+	// node holds the cluster-wide information about the node specific features.
+	// +nullable
+	Node *configv1.Node `json:"node"`
+
 	// dns holds the cluster dns details
 	// +nullable
 	DNS *configv1.DNS `json:"dns"`

pkg/controller/kubelet-config/kubelet_config_controller.go

@@ -359,6 +359,18 @@ func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig,
 		return nil, fmt.Errorf("could not merge feature gates: %v", err)
 	}
 
+	// (TODO) KubeletConfig can be modified based on the cgroupMode.
+	// modifying the kubelet's node-status-update-frequency based on the WorkerLatencyProfile
+	if cc.Spec.Node != nil {
+		switch cc.Spec.Node.Spec.WorkerLatencyProfile {
+		case configv1.MediumUpdateAverageReaction:
+			originalKubeConfig.NodeStatusUpdateFrequency = metav1.Duration{Duration: 20 * time.Second}
+		case configv1.LowUpdateSlowReaction:
+			originalKubeConfig.NodeStatusUpdateFrequency = metav1.Duration{Duration: 60 * time.Second}
+		default:
+			originalKubeConfig.NodeStatusUpdateFrequency = metav1.Duration{Duration: 10 * time.Second}
+		}
+	}
 	return originalKubeConfig, nil
 }
 

pkg/operator/bootstrap.go

@@ -100,6 +100,15 @@ func RenderBootstrap(
 		return err
 	}
 
+	// node provides information about the cgroup modes, worker latency profiles
+	// populating the node field with the default values.
+	spec.Node = &configv1.Node{
+		Spec: configv1.NodeSpec{
+			CgroupMode:           configv1.CgroupModeDefault,
+			WorkerLatencyProfile: configv1.DefaultUpdateDefaultReaction,
+		},
+	}
+
 	additionalTrustBundleData, err := ioutil.ReadFile(additionalTrustBundleFile)
 	if err != nil && !os.IsNotExist(err) {
 		return err

pkg/operator/operator.go

@@ -78,6 +78,7 @@ type Operator struct {
 	deployLister     appslisterv1.DeploymentLister
 	daemonsetLister  appslisterv1.DaemonSetLister
 	infraLister      configlistersv1.InfrastructureLister
+	confignodeLister configlistersv1.NodeLister
 	networkLister    configlistersv1.NetworkLister
 	mcoCmLister      corelisterv1.ConfigMapLister
 	clusterCmLister  corelisterv1.ConfigMapLister
@@ -90,6 +91,7 @@ type Operator struct {
 	deployListerSynced               cache.InformerSynced
 	daemonsetListerSynced            cache.InformerSynced
 	infraListerSynced                cache.InformerSynced
+	confignodeListerSynced           cache.InformerSynced
 	networkListerSynced              cache.InformerSynced
 	mcpListerSynced                  cache.InformerSynced
 	ccListerSynced                   cache.InformerSynced
@@ -128,6 +130,7 @@ func New(
 	mcoCmInformer,
 	clusterCmInfomer coreinformersv1.ConfigMapInformer,
 	infraInformer configinformersv1.InfrastructureInformer,
+	confignodeInformer configinformersv1.NodeInformer,
 	networkInformer configinformersv1.NetworkInformer,
 	proxyInformer configinformersv1.ProxyInformer,
 	dnsInformer configinformersv1.DNSInformer,
@@ -172,6 +175,7 @@ func New(
 		clusterRoleBindingInformer.Informer(),
 		mcoCmInformer.Informer(),
 		infraInformer.Informer(),
+		confignodeInformer.Informer(),
 		networkInformer.Informer(),
 		mcpInformer.Informer(),
 		proxyInformer.Informer(),
@@ -214,6 +218,8 @@ func New(
 	optr.daemonsetListerSynced = daemonsetInformer.Informer().HasSynced
 	optr.infraLister = infraInformer.Lister()
 	optr.infraListerSynced = infraInformer.Informer().HasSynced
+	optr.confignodeLister = confignodeInformer.Lister()
+	optr.confignodeListerSynced = confignodeInformer.Informer().HasSynced
 	optr.networkLister = networkInformer.Lister()
 	optr.networkListerSynced = networkInformer.Informer().HasSynced
 	optr.dnsLister = dnsInformer.Lister()
@@ -245,6 +251,7 @@ func (optr *Operator) Run(workers int, stopCh <-chan struct{}) {
 		optr.deployListerSynced,
 		optr.daemonsetListerSynced,
 		optr.infraListerSynced,
+		optr.confignodeListerSynced,
 		optr.mcoCmListerSynced,
 		optr.clusterCmListerSynced,
 		optr.serviceAccountInformerSynced,

pkg/operator/sync.go

@@ -271,6 +271,21 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig) error {
 		return err
 	}
 
+	node, err := optr.confignodeLister.Get("cluster")
+	if apierrors.IsNotFound(err) {
+		node = &configv1.Node{
+			Spec: configv1.NodeSpec{
+				CgroupMode:           configv1.CgroupModeDefault,
+				WorkerLatencyProfile: configv1.DefaultUpdateDefaultReaction,
+			},
+		}
+	} else if err != nil {
+		glog.V(2).Infof("%v", err)
+		return fmt.Errorf("could not fetch node object: %v", err)
+	}
+	if node != nil {
+		spec.Node = node
+	}
 	var trustBundle []byte
 	certPool := x509.NewCertPool()
 	// this is the generic trusted bundle for things like self-signed registries.

test/e2e-shared-tests/helpers.go

@@ -164,7 +164,7 @@ func assertNodeReachesState(t *testing.T, cs *framework.ClientSet, target corev1
 	maxWait := 5 * time.Minute
 
 	end, err := pollForResourceState(maxWait, func() (bool, error) {
-		node, err := cs.Nodes().Get(context.TODO(), target.Name, metav1.GetOptions{})
+		node, err := cs.CoreV1Interface.Nodes().Get(context.TODO(), target.Name, metav1.GetOptions{})
 		return stateFunc(*node), err
 	})
 

test/e2e/mcd_test.go

@@ -167,7 +167,7 @@ func TestKernelArguments(t *testing.T) {
 	workerMCP, err := cs.MachineConfigPools().Get(context.TODO(), "worker", metav1.GetOptions{})
 	require.Nil(t, err)
 	if err := wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
-		node, err := cs.Nodes().Get(context.TODO(), infraNode.Name, metav1.GetOptions{})
+		node, err := cs.CoreV1Interface.Nodes().Get(context.TODO(), infraNode.Name, metav1.GetOptions{})
 		require.Nil(t, err)
 		if node.Annotations[constants.DesiredMachineConfigAnnotationKey] != workerMCP.Spec.Configuration.Name {
 			return false, nil
@@ -254,7 +254,7 @@ func TestKernelType(t *testing.T) {
 	workerMCP, err := cs.MachineConfigPools().Get(context.TODO(), "worker", metav1.GetOptions{})
 	require.Nil(t, err)
 	if err := wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
-		node, err := cs.Nodes().Get(context.TODO(), infraNode.Name, metav1.GetOptions{})
+		node, err := cs.CoreV1Interface.Nodes().Get(context.TODO(), infraNode.Name, metav1.GetOptions{})
 		require.Nil(t, err)
 		if node.Annotations[constants.DesiredMachineConfigAnnotationKey] != workerMCP.Spec.Configuration.Name {
 			return false, nil
@@ -364,7 +364,7 @@ func TestExtensions(t *testing.T) {
 	workerMCP, err := cs.MachineConfigPools().Get(context.TODO(), "worker", metav1.GetOptions{})
 	require.Nil(t, err)
 	if err := wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
-		node, err := cs.Nodes().Get(context.TODO(), infraNode.Name, metav1.GetOptions{})
+		node, err := cs.CoreV1Interface.Nodes().Get(context.TODO(), infraNode.Name, metav1.GetOptions{})
 		require.Nil(t, err)
 		if node.Annotations[constants.DesiredMachineConfigAnnotationKey] != workerMCP.Spec.Configuration.Name {
 			return false, nil
@@ -489,7 +489,7 @@ func TestNoReboot(t *testing.T) {
 	workerMCP, err := cs.MachineConfigPools().Get(context.TODO(), "worker", metav1.GetOptions{})
 	require.Nil(t, err)
 	if err := wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
-		node, err := cs.Nodes().Get(context.TODO(), infraNode.Name, metav1.GetOptions{})
+		node, err := cs.CoreV1Interface.Nodes().Get(context.TODO(), infraNode.Name, metav1.GetOptions{})
 		require.Nil(t, err)
 		if node.Annotations[constants.DesiredMachineConfigAnnotationKey] != workerMCP.Spec.Configuration.Name {
 			return false, nil
@@ -696,7 +696,7 @@ func TestDontDeleteRPMFiles(t *testing.T) {
 	workerMCP, err := cs.MachineConfigPools().Get(context.TODO(), "worker", metav1.GetOptions{})
 	require.Nil(t, err)
 	if err := wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
-		node, err := cs.Nodes().Get(context.TODO(), infraNode.Name, metav1.GetOptions{})
+		node, err := cs.CoreV1Interface.Nodes().Get(context.TODO(), infraNode.Name, metav1.GetOptions{})
 		require.Nil(t, err)
 		if node.Annotations[constants.DesiredMachineConfigAnnotationKey] != workerMCP.Spec.Configuration.Name {
 			return false, nil
@@ -767,7 +767,7 @@ func TestIgn3Cfg(t *testing.T) {
 	workerMCP, err := cs.MachineConfigPools().Get(context.TODO(), "worker", metav1.GetOptions{})
 	require.Nil(t, err)
 	if err := wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
-		node, err := cs.Nodes().Get(context.TODO(), infraNode.Name, metav1.GetOptions{})
+		node, err := cs.CoreV1Interface.Nodes().Get(context.TODO(), infraNode.Name, metav1.GetOptions{})
 		require.Nil(t, err)
 		if node.Annotations[constants.DesiredMachineConfigAnnotationKey] != workerMCP.Spec.Configuration.Name {
 			return false, nil

test/helpers/utils.go

@@ -201,7 +201,7 @@ func WaitForPoolComplete(t *testing.T, cs *framework.ClientSet, pool, target str
 func WaitForNodeConfigChange(t *testing.T, cs *framework.ClientSet, node corev1.Node, mcName string) error {
 	startTime := time.Now()
 	err := wait.PollImmediate(2*time.Second, 20*time.Minute, func() (bool, error) {
-		n, err := cs.Nodes().Get(context.TODO(), node.Name, metav1.GetOptions{})
+		n, err := cs.CoreV1Interface.Nodes().Get(context.TODO(), node.Name, metav1.GetOptions{})
 		if err != nil {
 			return false, err
 		}
@@ -236,24 +236,24 @@ func LabelRandomNodeFromPool(t *testing.T, cs *framework.ClientSet, pool, label
 	infraNodeName := infraNode.Name
 
 	err = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
-		infraNode, err := cs.Nodes().Get(context.TODO(), infraNodeName, metav1.GetOptions{})
+		infraNode, err := cs.CoreV1Interface.Nodes().Get(context.TODO(), infraNodeName, metav1.GetOptions{})
 		if err != nil {
 			return err
 		}
 		infraNode.Labels[label] = ""
-		_, err = cs.Nodes().Update(context.TODO(), infraNode, metav1.UpdateOptions{})
+		_, err = cs.CoreV1Interface.Nodes().Update(context.TODO(), infraNode, metav1.UpdateOptions{})
 		return err
 	})
 	require.Nil(t, err, "unable to label worker node %s with infra: %s", infraNode.Name, err)
 
 	return func() {
 		err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
-			infraNode, err := cs.Nodes().Get(context.TODO(), infraNodeName, metav1.GetOptions{})
+			infraNode, err := cs.CoreV1Interface.Nodes().Get(context.TODO(), infraNodeName, metav1.GetOptions{})
 			if err != nil {
 				return err
 			}
 			delete(infraNode.Labels, label)
-			_, err = cs.Nodes().Update(context.TODO(), infraNode, metav1.UpdateOptions{})
+			_, err = cs.CoreV1Interface.Nodes().Update(context.TODO(), infraNode, metav1.UpdateOptions{})
 			return err
 		})
 		require.Nil(t, err, "unable to remove label from node %s: %s", infraNode.Name, err)
@@ -273,7 +273,7 @@ func GetNodesByRole(cs *framework.ClientSet, role string) ([]corev1.Node, error)
 	listOptions := metav1.ListOptions{
 		LabelSelector: labels.SelectorFromSet(labels.Set{fmt.Sprintf("node-role.kubernetes.io/%s", role): ""}).String(),
 	}
-	nodes, err := cs.Nodes().List(context.TODO(), listOptions)
+	nodes, err := cs.CoreV1Interface.Nodes().List(context.TODO(), listOptions)
 	if err != nil {
 		return nil, err
 	}