OTA-1174: upgrade status: Show resources in --details=health

pkg/cli/admin/upgrade/status/controlplane.go

@@ -53,11 +53,12 @@ const (
 )
 
 func coInsights(name string, available *v1.ClusterOperatorStatusCondition, degraded *v1.ClusterOperatorStatusCondition, evaluated time.Time) []updateInsight {
+	coGroupKind := scopeGroupKind{group: v1.GroupName, kind: "ClusterOperator"}
 	var insights []updateInsight
 	if available != nil && available.Status == v1.ConditionFalse && evaluated.After(available.LastTransitionTime.Time.Add(unavailableWarningThreshold)) {
 		insight := updateInsight{
 			startedAt: available.LastTransitionTime.Time,
-			scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: name}}},
+			scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coGroupKind, name: name}}},
 			impact: updateInsightImpact{
 				level:       warningImpactLevel,
 				impactType:  apiAvailabilityImpactType,
@@ -78,7 +79,7 @@ func coInsights(name string, available *v1.ClusterOperatorStatusCondition, degra
 	if degraded != nil && degraded.Status == v1.ConditionTrue && evaluated.After(degraded.LastTransitionTime.Time.Add(degradedWarningThreshold)) {
 		insight := updateInsight{
 			startedAt: degraded.LastTransitionTime.Time,
-			scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: name}}},
+			scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coGroupKind, name: name}}},
 			impact: updateInsightImpact{
 				level:       warningImpactLevel,
 				impactType:  apiAvailabilityImpactType,

pkg/cli/admin/upgrade/status/controlplane_test.go

@@ -133,6 +133,7 @@ var allowUnexportedInsightStructs = cmp.AllowUnexported(
 	updateInsight{},
 	updateInsightScope{},
 	scopeResource{},
+	scopeGroupKind{},
 	updateInsightImpact{},
 	updateInsightRemediation{},
 	updateHealthData{},
@@ -344,6 +345,7 @@ func TestAssessControlPlaneStatus_Duration(t *testing.T) {
 func TestCoInsights(t *testing.T) {
 	t.Parallel()
 	anchorTime := time.Now()
+	coKind := scopeGroupKind{group: configv1.GroupName, kind: "ClusterOperator"}
 	testCases := []struct {
 		name      string
 		available configv1.ClusterOperatorStatusCondition
@@ -393,7 +395,7 @@ func TestCoInsights(t *testing.T) {
 			expected: []updateInsight{
 				{
 					startedAt: anchorTime.Add(-unavailableWarningThreshold).Add(-time.Second),
-					scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}},
+					scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coKind, name: "testOperator"}}},
 					impact: updateInsightImpact{
 						level:       warningImpactLevel,
 						impactType:  apiAvailabilityImpactType,
@@ -406,7 +408,7 @@ func TestCoInsights(t *testing.T) {
 				},
 				{
 					startedAt: anchorTime.Add(-degradedWarningThreshold).Add(-time.Second),
-					scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}},
+					scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coKind, name: "testOperator"}}},
 					impact: updateInsightImpact{
 						level:       warningImpactLevel,
 						impactType:  apiAvailabilityImpactType,
@@ -438,7 +440,7 @@ func TestCoInsights(t *testing.T) {
 			expected: []updateInsight{
 				{
 					startedAt: anchorTime.Add(-unavailableErrorThreshold).Add(-time.Second),
-					scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}},
+					scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coKind, name: "testOperator"}}},
 					impact: updateInsightImpact{
 						level:       errorImpactLevel,
 						impactType:  apiAvailabilityImpactType,
@@ -451,7 +453,7 @@ func TestCoInsights(t *testing.T) {
 				},
 				{
 					startedAt: anchorTime.Add(-degradedErrorThreshold).Add(-time.Second),
-					scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}},
+					scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coKind, name: "testOperator"}}},
 					impact: updateInsightImpact{
 						level:       errorImpactLevel,
 						impactType:  apiAvailabilityImpactType,
@@ -480,7 +482,7 @@ func TestCoInsights(t *testing.T) {
 			expected: []updateInsight{
 				{
 					startedAt: anchorTime.Add(-unavailableErrorThreshold).Add(-time.Second),
-					scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}},
+					scope:     updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: coKind, name: "testOperator"}}},
 					impact: updateInsightImpact{
 						level:       errorImpactLevel,
 						impactType:  apiAvailabilityImpactType,

pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output

@@ -38,27 +38,35 @@ Message: Cluster Operator kube-apiserver is degraded (NodeController_MasterNodes
   Level:       Error
   Impact:      API Availability
   Reference:   https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md
+  Resources:
+    clusteroperators.config.openshift.io: kube-apiserver
   Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?)
 
 Message: Cluster Operator kube-controller-manager is degraded (NodeController_MasterNodesReady)
   Since:       58m18s
   Level:       Error
   Impact:      API Availability
   Reference:   https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md
+  Resources:
+    clusteroperators.config.openshift.io: kube-controller-manager
   Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?)
 
 Message: Cluster Operator kube-scheduler is degraded (NodeController_MasterNodesReady)
   Since:       58m18s
   Level:       Error
   Impact:      API Availability
   Reference:   https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md
+  Resources:
+    clusteroperators.config.openshift.io: kube-scheduler
   Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?)
 
 Message: Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady)
   Since:       58m38s
   Level:       Error
   Impact:      API Availability
   Reference:   https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md
+  Resources:
+    clusteroperators.config.openshift.io: etcd
   Description: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379"  Healthy:true Took:1.725492ms Error:<nil>} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379"  Healthy:true Took:1.542919ms Error:<nil>} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379"  Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}]
                , EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy
                , NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?)
@@ -68,4 +76,6 @@ Message: Cluster Operator control-plane-machine-set is unavailable (UnavailableR
   Level:       Error
   Impact:      API Availability
   Reference:   https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md
+  Resources:
+    clusteroperators.config.openshift.io: control-plane-machine-set
   Description: Missing 1 available replica(s)

pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output

@@ -39,3 +39,5 @@ SINCE     LEVEL   IMPACT             MESSAGE
 58m18s    Error   API Availability   Cluster Operator kube-scheduler is degraded (NodeController_MasterNodesReady)
 58m38s    Error   API Availability   Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady)
 1h0m17s   Error   API Availability   Cluster Operator control-plane-machine-set is unavailable (UnavailableReplicas)
+
+Run with --details=health for additional description and links to related online documentation

pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output

@@ -32,4 +32,6 @@ Message: Outdated nodes in a paused pool 'worker' will not be updated
   Level:       Warning
   Impact:      Update Stalled
   Reference:   https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues
+  Resources:
+    machineconfigpools.machineconfiguration.openshift.io: worker
   Description: Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator.

pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output

@@ -29,3 +29,5 @@ ip-10-0-99-40.us-east-2.compute.internal    Excluded     Paused   4.14.0    -
 = Update Health =
 SINCE   LEVEL     IMPACT           MESSAGE
 -       Warning   Update Stalled   Outdated nodes in a paused pool 'worker' will not be updated
+
+Run with --details=health for additional description and links to related online documentation

pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output

@@ -32,4 +32,6 @@ Message: Cluster Operator machine-config is unavailable (MachineConfigController
   Level:       Error
   Impact:      API Availability
   Reference:   https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md
+  Resources:
+    clusteroperators.config.openshift.io: machine-config
   Description: Cluster not available for [{operator 4.14.0-rc.3}]: ControllerConfig.machineconfiguration.openshift.io "machine-config-controller" is invalid: [status.controllerCertificates[0].notAfter: Required value, status.controllerCertificates[0].notBefore: Required value, status.controllerCertificates[1].notAfter: Required value, status.controllerCertificates[1].notBefore: Required value, status.controllerCertificates[2].notAfter: Required value, status.controllerCertificates[2].notBefore: Required value, status.controllerCertificates[3].notAfter: Required value, status.controllerCertificates[3].notBefore: Required value, status.controllerCertificates[4].notAfter: Required value, status.controllerCertificates[4].notBefore: Required value, status.controllerCertificates[5].notAfter: Required value, status.controllerCertificates[5].notBefore: Required value, status.controllerCertificates[6].notAfter: Required value, status.controllerCertificates[6].notBefore: Required value, status.controllerCertificates[7].notAfter: Required value, status.controllerCertificates[7].notBefore: Required value, status.controllerCertificates[8].notAfter: Required value, status.controllerCertificates[8].notBefore: Required value, status.controllerCertificates[9].notAfter: Required value, status.controllerCertificates[9].notBefore: Required value, <nil>: Invalid value: "null": some validation rules were not checked because the object was invalid; correct the existing errors to complete validation]

pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output

@@ -29,3 +29,5 @@ ip-10-0-99-40.us-east-2.compute.internal    Outdated     Pending   4.14.0-rc.3
 = Update Health =
 SINCE    LEVEL   IMPACT             MESSAGE
 20m24s   Error   API Availability   Cluster Operator machine-config is unavailable (MachineConfigControllerFailed)
+
+Run with --details=health for additional description and links to related online documentation