diff --git a/pkg/cli/admin/upgrade/status/controlplane.go b/pkg/cli/admin/upgrade/status/controlplane.go index 7dc5e84306..7ba002d23c 100644 --- a/pkg/cli/admin/upgrade/status/controlplane.go +++ b/pkg/cli/admin/upgrade/status/controlplane.go @@ -53,11 +53,12 @@ const ( ) func coInsights(name string, available *v1.ClusterOperatorStatusCondition, degraded *v1.ClusterOperatorStatusCondition, evaluated time.Time) []updateInsight { + coGroupKind := scopeGroupKind{group: v1.GroupName, kind: "ClusterOperator"} var insights []updateInsight if available != nil && available.Status == v1.ConditionFalse && evaluated.After(available.LastTransitionTime.Time.Add(unavailableWarningThreshold)) { insight := updateInsight{ startedAt: available.LastTransitionTime.Time, - scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: name}}}, + scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coGroupKind, name: name}}}, impact: updateInsightImpact{ level: warningImpactLevel, impactType: apiAvailabilityImpactType, @@ -78,7 +79,7 @@ func coInsights(name string, available *v1.ClusterOperatorStatusCondition, degra if degraded != nil && degraded.Status == v1.ConditionTrue && evaluated.After(degraded.LastTransitionTime.Time.Add(degradedWarningThreshold)) { insight := updateInsight{ startedAt: degraded.LastTransitionTime.Time, - scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: name}}}, + scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coGroupKind, name: name}}}, impact: updateInsightImpact{ level: warningImpactLevel, impactType: apiAvailabilityImpactType, diff --git a/pkg/cli/admin/upgrade/status/controlplane_test.go b/pkg/cli/admin/upgrade/status/controlplane_test.go index e117dc571e..76188241ce 100644 --- a/pkg/cli/admin/upgrade/status/controlplane_test.go +++ b/pkg/cli/admin/upgrade/status/controlplane_test.go @@ -133,6 +133,7 @@ var allowUnexportedInsightStructs = cmp.AllowUnexported( updateInsight{}, updateInsightScope{}, scopeResource{}, + scopeGroupKind{}, updateInsightImpact{}, updateInsightRemediation{}, updateHealthData{}, @@ -344,6 +345,7 @@ func TestAssessControlPlaneStatus_Duration(t *testing.T) { func TestCoInsights(t *testing.T) { t.Parallel() anchorTime := time.Now() + coKind := scopeGroupKind{group: configv1.GroupName, kind: "ClusterOperator"} testCases := []struct { name string available configv1.ClusterOperatorStatusCondition @@ -393,7 +395,7 @@ func TestCoInsights(t *testing.T) { expected: []updateInsight{ { startedAt: anchorTime.Add(-unavailableWarningThreshold).Add(-time.Second), - scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, + scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coKind, name: "testOperator"}}}, impact: updateInsightImpact{ level: warningImpactLevel, impactType: apiAvailabilityImpactType, @@ -406,7 +408,7 @@ func TestCoInsights(t *testing.T) { }, { startedAt: anchorTime.Add(-degradedWarningThreshold).Add(-time.Second), - scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, + scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coKind, name: "testOperator"}}}, impact: updateInsightImpact{ level: warningImpactLevel, impactType: apiAvailabilityImpactType, @@ -438,7 +440,7 @@ func TestCoInsights(t *testing.T) { expected: []updateInsight{ { startedAt: anchorTime.Add(-unavailableErrorThreshold).Add(-time.Second), - scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, + scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coKind, name: "testOperator"}}}, impact: updateInsightImpact{ level: errorImpactLevel, impactType: apiAvailabilityImpactType, @@ -451,7 +453,7 @@ func TestCoInsights(t *testing.T) { }, { startedAt: anchorTime.Add(-degradedErrorThreshold).Add(-time.Second), - scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, + scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coKind, name: "testOperator"}}}, impact: updateInsightImpact{ level: errorImpactLevel, impactType: apiAvailabilityImpactType, @@ -480,7 +482,7 @@ func TestCoInsights(t *testing.T) { expected: []updateInsight{ { startedAt: anchorTime.Add(-unavailableErrorThreshold).Add(-time.Second), - scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, + scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coKind, name: "testOperator"}}}, impact: updateInsightImpact{ level: errorImpactLevel, impactType: apiAvailabilityImpactType, diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output index 31aae200ed..cebe8c8a8e 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output @@ -38,6 +38,8 @@ Message: Cluster Operator kube-apiserver is degraded (NodeController_MasterNodes Level: Error Impact: API Availability Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Resources: + clusteroperators.config.openshift.io: kube-apiserver Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) Message: Cluster Operator kube-controller-manager is degraded (NodeController_MasterNodesReady) @@ -45,6 +47,8 @@ Message: Cluster Operator kube-controller-manager is degraded (NodeController_Ma Level: Error Impact: API Availability Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Resources: + clusteroperators.config.openshift.io: kube-controller-manager Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) Message: Cluster Operator kube-scheduler is degraded (NodeController_MasterNodesReady) @@ -52,6 +56,8 @@ Message: Cluster Operator kube-scheduler is degraded (NodeController_MasterNodes Level: Error Impact: API Availability Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Resources: + clusteroperators.config.openshift.io: kube-scheduler Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) Message: Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady) @@ -59,6 +65,8 @@ Message: Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpo Level: Error Impact: API Availability Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Resources: + clusteroperators.config.openshift.io: etcd Description: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] , EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy , NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) @@ -68,4 +76,6 @@ Message: Cluster Operator control-plane-machine-set is unavailable (UnavailableR Level: Error Impact: API Availability Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md + Resources: + clusteroperators.config.openshift.io: control-plane-machine-set Description: Missing 1 available replica(s) diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output index b8b40be6d1..2b5c60f752 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output @@ -39,3 +39,5 @@ SINCE LEVEL IMPACT MESSAGE 58m18s Error API Availability Cluster Operator kube-scheduler is degraded (NodeController_MasterNodesReady) 58m38s Error API Availability Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady) 1h0m17s Error API Availability Cluster Operator control-plane-machine-set is unavailable (UnavailableReplicas) + +Run with --details=health for additional description and links to related online documentation diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output index d11c6be945..55b424ac7f 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output @@ -32,4 +32,6 @@ Message: Outdated nodes in a paused pool 'worker' will not be updated Level: Warning Impact: Update Stalled Reference: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues + Resources: + machineconfigpools.machineconfiguration.openshift.io: worker Description: Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator. diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output index 947fddfa10..3ccb766ad3 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output @@ -29,3 +29,5 @@ ip-10-0-99-40.us-east-2.compute.internal Excluded Paused 4.14.0 - = Update Health = SINCE LEVEL IMPACT MESSAGE - Warning Update Stalled Outdated nodes in a paused pool 'worker' will not be updated + +Run with --details=health for additional description and links to related online documentation diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output index fff5957040..bd340a0301 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output @@ -32,4 +32,6 @@ Message: Cluster Operator machine-config is unavailable (MachineConfigController Level: Error Impact: API Availability Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md + Resources: + clusteroperators.config.openshift.io: machine-config Description: Cluster not available for [{operator 4.14.0-rc.3}]: ControllerConfig.machineconfiguration.openshift.io "machine-config-controller" is invalid: [status.controllerCertificates[0].notAfter: Required value, status.controllerCertificates[0].notBefore: Required value, status.controllerCertificates[1].notAfter: Required value, status.controllerCertificates[1].notBefore: Required value, status.controllerCertificates[2].notAfter: Required value, status.controllerCertificates[2].notBefore: Required value, status.controllerCertificates[3].notAfter: Required value, status.controllerCertificates[3].notBefore: Required value, status.controllerCertificates[4].notAfter: Required value, status.controllerCertificates[4].notBefore: Required value, status.controllerCertificates[5].notAfter: Required value, status.controllerCertificates[5].notBefore: Required value, status.controllerCertificates[6].notAfter: Required value, status.controllerCertificates[6].notBefore: Required value, status.controllerCertificates[7].notAfter: Required value, status.controllerCertificates[7].notBefore: Required value, status.controllerCertificates[8].notAfter: Required value, status.controllerCertificates[8].notBefore: Required value, status.controllerCertificates[9].notAfter: Required value, status.controllerCertificates[9].notBefore: Required value, : Invalid value: "null": some validation rules were not checked because the object was invalid; correct the existing errors to complete validation] diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output index 592661bc03..2b1c8b89bf 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output @@ -29,3 +29,5 @@ ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 = Update Health = SINCE LEVEL IMPACT MESSAGE 20m24s Error API Availability Cluster Operator machine-config is unavailable (MachineConfigControllerFailed) + +Run with --details=health for additional description and links to related online documentation diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output index 5efe8290ae..f83ba4e2cb 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output @@ -28,13 +28,13 @@ Worker Status: 59 Total, 46 Available, 5 Progressing, 36 Outdated, 12 Draining Worker Pool Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE -build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information build0-gstfj-ci-tests-worker-c-jq5rk Unavailable Updated 4.16.0-ec.3 - Node is unavailable build0-gstfj-ci-tests-worker-c-2kz4m Progressing Draining 4.16.0-ec.2 +30m build0-gstfj-ci-tests-worker-c-55hpj Progressing Draining 4.16.0-ec.2 +30m @@ -94,6 +94,8 @@ Message: Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded Level: Error Impact: Update Stalled Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Resources: + nodes: build0-gstfj-ci-prowjobs-worker-b-9lztv Description: failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information Message: Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded @@ -101,6 +103,8 @@ Message: Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded Level: Error Impact: Update Stalled Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Resources: + nodes: build0-gstfj-ci-prowjobs-worker-b-bg9f5 Description: failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information Message: Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded @@ -108,6 +112,8 @@ Message: Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded Level: Error Impact: Update Stalled Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Resources: + nodes: build0-gstfj-ci-prowjobs-worker-b-mrxwn Description: failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information Message: Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded @@ -115,6 +121,8 @@ Message: Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded Level: Error Impact: Update Stalled Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Resources: + nodes: build0-gstfj-ci-tests-worker-b-4h7pn Description: failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information Message: Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded @@ -122,6 +130,8 @@ Message: Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded Level: Error Impact: Update Stalled Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Resources: + nodes: build0-gstfj-ci-tests-worker-b-jv5bg Description: failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information Message: Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded @@ -129,6 +139,8 @@ Message: Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded Level: Error Impact: Update Stalled Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Resources: + nodes: build0-gstfj-ci-tests-worker-b-kj6gk Description: failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information Message: Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded @@ -136,6 +148,8 @@ Message: Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded Level: Error Impact: Update Stalled Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Resources: + nodes: build0-gstfj-ci-tests-worker-c-dcz9p Description: failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information Message: Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable @@ -143,4 +157,6 @@ Message: Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable Level: Warning Impact: Update Speed Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Resources: + nodes: build0-gstfj-ci-tests-worker-c-jq5rk Description: Node is unavailable diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output index 0f26fcc43f..ca735ee8e2 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output @@ -28,13 +28,13 @@ Worker Status: 59 Total, 46 Available, 5 Progressing, 36 Outdated, 12 Draining Worker Pool Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE -build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information build0-gstfj-ci-tests-worker-c-jq5rk Unavailable Updated 4.16.0-ec.3 - Node is unavailable build0-gstfj-ci-tests-worker-c-2kz4m Progressing Draining 4.16.0-ec.2 +30m build0-gstfj-ci-tests-worker-c-55hpj Progressing Draining 4.16.0-ec.2 +30m @@ -52,3 +52,5 @@ SINCE LEVEL IMPACT MESSAGE - Error Update Stalled Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded - Error Update Stalled Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded - Warning Update Speed Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable + +Run with --details=health for additional description and links to related online documentation diff --git a/pkg/cli/admin/upgrade/status/health.go b/pkg/cli/admin/upgrade/status/health.go index 0d53c38d94..ef047b0c4a 100644 --- a/pkg/cli/admin/upgrade/status/health.go +++ b/pkg/cli/admin/upgrade/status/health.go @@ -16,16 +16,20 @@ const ( scopeTypeWorkerPool scopeType = "WorkerPool" ) -type allowedScopeKind string +type scopeGroupKind struct { + group string + kind string +} -const ( - scopeKindClusterOperator allowedScopeKind = "ClusterOperator" - scopeKindNode allowedScopeKind = "Node" - scopeKindMachineConfigPool allowedScopeKind = "MachineConfigPool" -) +func (r scopeResource) namespacedName() string { + if r.namespace == "" { + return r.name + } + return fmt.Sprintf("%s/%s", r.namespace, r.name) +} type scopeResource struct { - kind allowedScopeKind + kind scopeGroupKind namespace string name string } @@ -194,19 +198,48 @@ type displayItem struct { message string description string reference string + + resourceKindPad int + // resourceKinds contains keys of resources map + resourceKinds []string + // Nodes: node1, node2, node3 + resources map[string][]string } func (i *updateHealthData) Write(w io.Writer, detailed bool) error { _, _ = w.Write([]byte("= Update Health =\n")) displayData := make([]displayItem, 0, len(i.insights)) for _, insight := range i.insights { + var resourceKinds []string + var resources map[string][]string + var resourceKindPad int + for _, resource := range insight.scope.resources { + if resources == nil { + resources = map[string][]string{} + } + + kind := fmt.Sprintf("%ss", strings.ToLower(resource.kind.kind)) + if resource.kind.group != "" { + kind = fmt.Sprintf("%s.%s", kind, resource.kind.group) + } + kind = fmt.Sprintf("%s: ", kind) + if len(kind) > resourceKindPad { + resourceKindPad = len(kind) + } + resourceKinds = append(resourceKinds, kind) + resources[kind] = append(resources[kind], resource.namespacedName()) + } + displayData = append(displayData, displayItem{ - since: stringSince(i, insight), - level: insight.impact.level.String(), - impact: string(insight.impact.impactType), - message: insight.impact.summary, - description: insight.impact.description, - reference: insight.remediation.reference, + since: stringSince(i, insight), + level: insight.impact.level.String(), + impact: string(insight.impact.impactType), + message: insight.impact.summary, + description: insight.impact.description, + reference: insight.remediation.reference, + resourceKindPad: resourceKindPad, + resourceKinds: resourceKinds, + resources: resources, }) } @@ -229,10 +262,20 @@ func detailedOutput(w io.Writer, items []displayItem) { _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Reference:", item.reference))) // Respect the " Description: " indentation when description has linebreaks item.description = strings.ReplaceAll(item.description, "\n", fmt.Sprintf("\n%s, ", strings.Repeat(" ", pad+2))) + + if len(item.resourceKinds) > 0 { + _, _ = w.Write([]byte(fmt.Sprintf(" %s\n", "Resources:"))) + sort.Strings(item.resourceKinds) + for _, kind := range item.resourceKinds { + sort.Strings(item.resources[kind]) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", item.resourceKindPad, kind, strings.Join(item.resources[kind], ", ")))) + } + } + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Description:", item.description))) + if len(items) > i+1 { _, _ = w.Write([]byte("\n")) - } } } @@ -247,4 +290,8 @@ func tabulatedOutput(w io.Writer, items []displayItem) { _, _ = tabw.Write([]byte(item.message + "\n")) } _ = tabw.Flush() + + if len(items) > 1 || items[0].level != infoImpactLevel.String() { + _, _ = w.Write([]byte("\nRun with --details=health for additional description and links to related online documentation\n")) + } } diff --git a/pkg/cli/admin/upgrade/status/health_test.go b/pkg/cli/admin/upgrade/status/health_test.go index 2114da8fb9..1cf4b73e99 100644 --- a/pkg/cli/admin/upgrade/status/health_test.go +++ b/pkg/cli/admin/upgrade/status/health_test.go @@ -191,7 +191,8 @@ func TestUpdateHealthData_Write(t *testing.T) { "5s Error API Availability Something that broke API and happened recently\n" + "20s Error Cluster Capacity Something that limits cluster capacity\n" + "10s Warning Update Speed Something that slows the update\n" + - "0s Info None Something with no impact that happened right now\n", + "0s Info None Something with no impact that happened right now\n\n" + + "Run with --details=health for additional description and links to related online documentation\n", }, { name: "detailed", diff --git a/pkg/cli/admin/upgrade/status/workerpool.go b/pkg/cli/admin/upgrade/status/workerpool.go index 5661056c61..56fbf7ea92 100644 --- a/pkg/cli/admin/upgrade/status/workerpool.go +++ b/pkg/cli/admin/upgrade/status/workerpool.go @@ -135,6 +135,14 @@ func selectNodesFromPool(pool mcfgv1.MachineConfigPool, allNodes []corev1.Node) return res, nil } +func ellipsizeNames(message string, name string) string { + if len(name) < 8 { + return message + } + + return strings.Replace(message, name, "", -1) +} + func assessNodesStatus(cv *configv1.ClusterVersion, pool mcfgv1.MachineConfigPool, nodes []corev1.Node, machineConfigs []mcfgv1.MachineConfig) ([]nodeDisplayData, []updateInsight) { var nodesStatusData []nodeDisplayData var insights []updateInsight @@ -198,7 +206,7 @@ func assessNodesStatus(cv *configv1.ClusterVersion, pool mcfgv1.MachineConfigPoo Assessment: assessment, Estimate: estimate, Phase: phase, - Message: message, + Message: ellipsizeNames(message, node.Name), Version: currentVersion, isUnavailable: isUnavailable, isDegraded: isDegraded, @@ -324,12 +332,14 @@ func nodeInsights(pool mcfgv1.MachineConfigPool, node corev1.Node, reason string if pool.Name == "master" { scope = scopeTypeControlPlane } + nodeGvk := node.GroupVersionKind() + nodeGroupKind := scopeGroupKind{group: nodeGvk.Group, kind: nodeGvk.Kind} if isUnavailable && !isUpdating { insights = append(insights, updateInsight{ startedAt: time.Time{}, scope: updateInsightScope{ scopeType: scope, - resources: []scopeResource{{kind: scopeKindNode, name: node.Name}}, + resources: []scopeResource{{kind: nodeGroupKind, name: node.Name}}, }, impact: updateInsightImpact{ level: warningImpactLevel, @@ -347,7 +357,7 @@ func nodeInsights(pool mcfgv1.MachineConfigPool, node corev1.Node, reason string startedAt: time.Time{}, scope: updateInsightScope{ scopeType: scope, - resources: []scopeResource{{kind: scopeKindNode, name: node.Name}}, + resources: []scopeResource{{kind: nodeGroupKind, name: node.Name}}, }, impact: updateInsightImpact{ level: errorImpactLevel, @@ -434,11 +444,12 @@ func machineConfigPoolInsights(poolDisplay poolDisplayData, pool mcfgv1.MachineC // TODO: Only generate this insight if the pool has some work remaining that will not finish // Depends on how MCO actually works: will it stop updating a node that already started e.g. draining?) if poolDisplay.NodesOverview.Excluded > 0 && pool.Spec.Paused { + poolGvk := pool.GroupVersionKind() insights = append(insights, updateInsight{ startedAt: time.Time{}, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, - resources: []scopeResource{{kind: scopeKindMachineConfigPool, name: pool.Name}}, + resources: []scopeResource{{kind: scopeGroupKind{group: poolGvk.Group, kind: poolGvk.Kind}, name: pool.Name}}, }, impact: updateInsightImpact{ level: warningImpactLevel, diff --git a/pkg/cli/admin/upgrade/status/workerpool_test.go b/pkg/cli/admin/upgrade/status/workerpool_test.go index 0f1f0e5d2d..03a4777833 100644 --- a/pkg/cli/admin/upgrade/status/workerpool_test.go +++ b/pkg/cli/admin/upgrade/status/workerpool_test.go @@ -14,7 +14,11 @@ import ( "github.com/openshift/oc/pkg/cli/admin/upgrade/status/mco" ) -var allowUnexportedWorkerPools = cmp.AllowUnexported(nodeDisplayData{}, nodesOverviewDisplayData{}, poolDisplayData{}) +var allowUnexportedWorkerPools = cmp.AllowUnexported( + nodeDisplayData{}, + nodesOverviewDisplayData{}, + poolDisplayData{}, +) type mcpBuilder struct { machineConfigPool mcfgv1.MachineConfigPool @@ -23,6 +27,10 @@ type mcpBuilder struct { func mcp(name string) *mcpBuilder { return &mcpBuilder{ machineConfigPool: mcfgv1.MachineConfigPool{ + TypeMeta: v1.TypeMeta{ + Kind: "MachineConfigPool", + APIVersion: mcfgv1.GroupVersion.String(), + }, ObjectMeta: v1.ObjectMeta{ Name: name, Labels: map[string]string{ @@ -78,6 +86,9 @@ type nodeBuilder struct { func node(name string) *nodeBuilder { return &nodeBuilder{ node: corev1.Node{ + TypeMeta: v1.TypeMeta{ + Kind: "Node", + }, ObjectMeta: v1.ObjectMeta{ Name: name, Annotations: map[string]string{}, @@ -229,7 +240,10 @@ func (n *nodeBuilder) degraded_draining(currentConfig, desiredConfig, reason str return n.annotated(annotations).unavailable() } +var nodeGroupKind = scopeGroupKind{group: corev1.GroupName, kind: "Node"} + func Test_assessNodesStatus(t *testing.T) { + oldOCPVersion := "3.10.0" newOCPVersion := "4.16.0" mcOld := mc("old").version(oldOCPVersion).machineConfig @@ -471,7 +485,7 @@ func Test_assessNodesStatus(t *testing.T) { }, scope: updateInsightScope{ scopeType: scopeTypeControlPlane, - resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, + resources: []scopeResource{{kind: nodeGroupKind, name: "a"}}, }, remediation: updateInsightRemediation{ reference: "https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", @@ -513,7 +527,7 @@ func Test_assessNodesStatus(t *testing.T) { }, scope: updateInsightScope{ scopeType: scopeTypeControlPlane, - resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, + resources: []scopeResource{{kind: nodeGroupKind, name: "a"}}, }, remediation: updateInsightRemediation{ reference: "https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", @@ -692,7 +706,7 @@ func Test_nodeInsights(t *testing.T) { }, scope: updateInsightScope{ scopeType: scopeTypeControlPlane, - resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, + resources: []scopeResource{{kind: nodeGroupKind, name: "a"}}, }, remediation: updateInsightRemediation{ reference: "https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", @@ -718,7 +732,7 @@ func Test_nodeInsights(t *testing.T) { }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, - resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, + resources: []scopeResource{{kind: nodeGroupKind, name: "a"}}, }, remediation: updateInsightRemediation{ reference: "https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", @@ -746,7 +760,7 @@ func Test_nodeInsights(t *testing.T) { }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, - resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, + resources: []scopeResource{{kind: nodeGroupKind, name: "a"}}, }, remediation: updateInsightRemediation{ reference: "https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", @@ -765,6 +779,11 @@ func Test_nodeInsights(t *testing.T) { } } +var mcpGroupKind = scopeGroupKind{ + group: mcfgv1.GroupName, + kind: "MachineConfigPool", +} + func Test_assessMachineConfigPool(t *testing.T) { oldOCPVersion := "3.10.0" newOCPVersion := "4.16.0" @@ -984,7 +1003,7 @@ func Test_assessMachineConfigPool(t *testing.T) { }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, - resources: []scopeResource{{kind: scopeKindMachineConfigPool, name: "worker"}}, + resources: []scopeResource{{kind: mcpGroupKind, name: "worker"}}, }, remediation: updateInsightRemediation{ reference: "https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues", @@ -1080,7 +1099,7 @@ func Test_machineConfigPoolInsights(t *testing.T) { }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, - resources: []scopeResource{{kind: scopeKindMachineConfigPool, name: "worker"}}, + resources: []scopeResource{{kind: mcpGroupKind, name: "worker"}}, }, remediation: updateInsightRemediation{ "https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues",