diff --git a/pkg/cli/admin/upgrade/status/controlplane.go b/pkg/cli/admin/upgrade/status/controlplane.go index 3ef4e7b994..a78ec102b5 100644 --- a/pkg/cli/admin/upgrade/status/controlplane.go +++ b/pkg/cli/admin/upgrade/status/controlplane.go @@ -48,10 +48,16 @@ func coInsights(name string, available v1.ClusterOperatorStatusCondition, degrad startedAt: available.LastTransitionTime.Time, scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: name}}}, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: apiAvailabilityImpactType, - summary: fmt.Sprintf("Cluster Operator %s is unavailable | %s: %s", name, available.Reason, strings.ReplaceAll(available.Message, "\n", ` // `)), + level: warningImpactLevel, + impactType: apiAvailabilityImpactType, + summary: fmt.Sprintf("Cluster Operator %s is unavailable (%s)", name, available.Reason), + description: available.Message, }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"}, + } + if available.Message == "" { + // Backfill the description if CO doesn't provide one + insight.impact.description = "" } if evaluated.After(available.LastTransitionTime.Time.Add(unavailableErrorThreshold)) { insight.impact.level = errorImpactLevel @@ -63,10 +69,16 @@ func coInsights(name string, available v1.ClusterOperatorStatusCondition, degrad startedAt: degraded.LastTransitionTime.Time, scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: name}}}, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: apiAvailabilityImpactType, - summary: fmt.Sprintf("Cluster Operator %s is degraded | %s: %s", name, degraded.Reason, strings.ReplaceAll(degraded.Message, "\n", ` // `)), + level: warningImpactLevel, + impactType: apiAvailabilityImpactType, + summary: fmt.Sprintf("Cluster Operator %s is degraded (%s)", name, degraded.Reason), + description: degraded.Message, }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md"}, + } + if degraded.Message == "" { + // Backfill the description if CO doesn't provide one + insight.impact.description = "" } if evaluated.After(degraded.LastTransitionTime.Time.Add(degradedErrorThreshold)) { insight.impact.level = errorImpactLevel diff --git a/pkg/cli/admin/upgrade/status/controlplane_test.go b/pkg/cli/admin/upgrade/status/controlplane_test.go index 9e4eb40044..f5c0ee9f69 100644 --- a/pkg/cli/admin/upgrade/status/controlplane_test.go +++ b/pkg/cli/admin/upgrade/status/controlplane_test.go @@ -129,7 +129,14 @@ var cvFixture = configv1.ClusterVersion{ }, } -var allowUnexportedInsightStructs = cmp.AllowUnexported(updateInsight{}, updateInsightScope{}, scopeResource{}, updateInsightImpact{}) +var allowUnexportedInsightStructs = cmp.AllowUnexported( + updateInsight{}, + updateInsightScope{}, + scopeResource{}, + updateInsightImpact{}, + updateInsightRemediation{}, + updateHealthData{}, +) func TestAssessControlPlaneStatus_Operators(t *testing.T) { testCases := []struct { @@ -404,18 +411,26 @@ func TestCoInsights(t *testing.T) { startedAt: anchorTime.Add(-unavailableWarningThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Cluster Operator testOperator is unavailable | Broken: Operator is broken", + level: warningImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Cluster Operator testOperator is unavailable (Broken)", + description: "Operator is broken", + }, + remediation: updateInsightRemediation{ + reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md", }, }, { startedAt: anchorTime.Add(-degradedWarningThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Cluster Operator testOperator is degraded | Slow: Networking is hard", + level: warningImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Cluster Operator testOperator is degraded (Slow)", + description: "Networking is hard", + }, + remediation: updateInsightRemediation{ + reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md", }, }, }, @@ -441,24 +456,32 @@ func TestCoInsights(t *testing.T) { startedAt: anchorTime.Add(-unavailableErrorThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Cluster Operator testOperator is unavailable | Broken: Operator is broken", + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Cluster Operator testOperator is unavailable (Broken)", + description: "Operator is broken", + }, + remediation: updateInsightRemediation{ + reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md", }, }, { startedAt: anchorTime.Add(-degradedErrorThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Cluster Operator testOperator is degraded | Slow: Networking is hard", + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Cluster Operator testOperator is degraded (Slow)", + description: "Networking is hard", + }, + remediation: updateInsightRemediation{ + reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md", }, }, }, }, { - name: "insights flatten linebreaks in messages", + name: "insights do not flatten linebreaks in messages", available: configv1.ClusterOperatorStatusCondition{ Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse, @@ -475,9 +498,13 @@ func TestCoInsights(t *testing.T) { startedAt: anchorTime.Add(-unavailableErrorThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: apiAvailabilityImpactType, - summary: `Cluster Operator testOperator is unavailable | Broken: Operator is broken // and message has linebreaks`, + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: `Cluster Operator testOperator is unavailable (Broken)`, + description: "Operator is broken\nand message has linebreaks", + }, + remediation: updateInsightRemediation{ + reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md", }, }, }, diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output index 35aa29b7c7..33b9107de1 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output @@ -33,9 +33,39 @@ ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? = Update Health = -SINCE LEVEL IMPACT MESSAGE -58m18s Error API Availability Cluster Operator kube-apiserver is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -58m18s Error API Availability Cluster Operator kube-controller-manager is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -58m18s Error API Availability Cluster Operator kube-scheduler is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -58m38s Error API Availability Cluster Operator etcd is degraded | EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] // EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy // NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -1h0m17s Error API Availability Cluster Operator control-plane-machine-set is unavailable | UnavailableReplicas: Missing 1 available replica(s) +Message: Cluster Operator kube-apiserver is degraded (NodeController_MasterNodesReady) + Since: 58m18s + Level: Error + Impact: API Availability + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) + +Message: Cluster Operator kube-controller-manager is degraded (NodeController_MasterNodesReady) + Since: 58m18s + Level: Error + Impact: API Availability + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) + +Message: Cluster Operator kube-scheduler is degraded (NodeController_MasterNodesReady) + Since: 58m18s + Level: Error + Impact: API Availability + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) + +Message: Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady) + Since: 58m38s + Level: Error + Impact: API Availability + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Description: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] + , EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy + , NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) + +Message: Cluster Operator control-plane-machine-set is unavailable (UnavailableReplicas) + Since: 1h0m17s + Level: Error + Impact: API Availability + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md + Description: Missing 1 available replica(s) diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output index 35aa29b7c7..e832e2ca1a 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output @@ -34,8 +34,8 @@ ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 = Update Health = SINCE LEVEL IMPACT MESSAGE -58m18s Error API Availability Cluster Operator kube-apiserver is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -58m18s Error API Availability Cluster Operator kube-controller-manager is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -58m18s Error API Availability Cluster Operator kube-scheduler is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -58m38s Error API Availability Cluster Operator etcd is degraded | EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] // EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy // NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -1h0m17s Error API Availability Cluster Operator control-plane-machine-set is unavailable | UnavailableReplicas: Missing 1 available replica(s) +58m18s Error API Availability Cluster Operator kube-apiserver is degraded (NodeController_MasterNodesReady) +58m18s Error API Availability Cluster Operator kube-controller-manager is degraded (NodeController_MasterNodesReady) +58m18s Error API Availability Cluster Operator kube-scheduler is degraded (NodeController_MasterNodesReady) +58m38s Error API Availability Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady) +1h0m17s Error API Availability Cluster Operator control-plane-machine-set is unavailable (UnavailableReplicas) diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output index dd22bdd9e8..8a4a6482b4 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output @@ -27,5 +27,9 @@ ip-10-0-4-159.us-east-2.compute.internal Excluded Paused 4.14.0 - ip-10-0-99-40.us-east-2.compute.internal Excluded Paused 4.14.0 - = Update Health = -SINCE LEVEL IMPACT MESSAGE -- Warning Update Stalled Worker pool worker is paused | Outdated nodes in a paused pool will not be updated. +Message: Outdated nodes in a paused pool 'worker' will not be updated + Since: - + Level: Warning + Impact: Update Stalled + Reference: https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues + Description: Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator. diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output index dd22bdd9e8..efa519b55d 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output @@ -28,4 +28,4 @@ ip-10-0-99-40.us-east-2.compute.internal Excluded Paused 4.14.0 - = Update Health = SINCE LEVEL IMPACT MESSAGE -- Warning Update Stalled Worker pool worker is paused | Outdated nodes in a paused pool will not be updated. +- Warning Update Stalled Outdated nodes in a paused pool 'worker' will not be updated diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output index af5e71c194..0bcc9a2452 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output @@ -27,5 +27,9 @@ ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? = Update Health = -SINCE LEVEL IMPACT MESSAGE -20m24s Error API Availability Cluster Operator machine-config is unavailable | MachineConfigControllerFailed: Cluster not available for [{operator 4.14.0-rc.3}]: ControllerConfig.machineconfiguration.openshift.io "machine-config-controller" is invalid: [status.controllerCertificates[0].notAfter: Required value, status.controllerCertificates[0].notBefore: Required value, status.controllerCertificates[1].notAfter: Required value, status.controllerCertificates[1].notBefore: Required value, status.controllerCertificates[2].notAfter: Required value, status.controllerCertificates[2].notBefore: Required value, status.controllerCertificates[3].notAfter: Required value, status.controllerCertificates[3].notBefore: Required value, status.controllerCertificates[4].notAfter: Required value, status.controllerCertificates[4].notBefore: Required value, status.controllerCertificates[5].notAfter: Required value, status.controllerCertificates[5].notBefore: Required value, status.controllerCertificates[6].notAfter: Required value, status.controllerCertificates[6].notBefore: Required value, status.controllerCertificates[7].notAfter: Required value, status.controllerCertificates[7].notBefore: Required value, status.controllerCertificates[8].notAfter: Required value, status.controllerCertificates[8].notBefore: Required value, status.controllerCertificates[9].notAfter: Required value, status.controllerCertificates[9].notBefore: Required value, : Invalid value: "null": some validation rules were not checked because the object was invalid; correct the existing errors to complete validation] +Message: Cluster Operator machine-config is unavailable (MachineConfigControllerFailed) + Since: 20m24s + Level: Error + Impact: API Availability + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md + Description: Cluster not available for [{operator 4.14.0-rc.3}]: ControllerConfig.machineconfiguration.openshift.io "machine-config-controller" is invalid: [status.controllerCertificates[0].notAfter: Required value, status.controllerCertificates[0].notBefore: Required value, status.controllerCertificates[1].notAfter: Required value, status.controllerCertificates[1].notBefore: Required value, status.controllerCertificates[2].notAfter: Required value, status.controllerCertificates[2].notBefore: Required value, status.controllerCertificates[3].notAfter: Required value, status.controllerCertificates[3].notBefore: Required value, status.controllerCertificates[4].notAfter: Required value, status.controllerCertificates[4].notBefore: Required value, status.controllerCertificates[5].notAfter: Required value, status.controllerCertificates[5].notBefore: Required value, status.controllerCertificates[6].notAfter: Required value, status.controllerCertificates[6].notBefore: Required value, status.controllerCertificates[7].notAfter: Required value, status.controllerCertificates[7].notBefore: Required value, status.controllerCertificates[8].notAfter: Required value, status.controllerCertificates[8].notBefore: Required value, status.controllerCertificates[9].notAfter: Required value, status.controllerCertificates[9].notBefore: Required value, : Invalid value: "null": some validation rules were not checked because the object was invalid; correct the existing errors to complete validation] diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output index af5e71c194..1f241b314c 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output @@ -28,4 +28,4 @@ ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 = Update Health = SINCE LEVEL IMPACT MESSAGE -20m24s Error API Availability Cluster Operator machine-config is unavailable | MachineConfigControllerFailed: Cluster not available for [{operator 4.14.0-rc.3}]: ControllerConfig.machineconfiguration.openshift.io "machine-config-controller" is invalid: [status.controllerCertificates[0].notAfter: Required value, status.controllerCertificates[0].notBefore: Required value, status.controllerCertificates[1].notAfter: Required value, status.controllerCertificates[1].notBefore: Required value, status.controllerCertificates[2].notAfter: Required value, status.controllerCertificates[2].notBefore: Required value, status.controllerCertificates[3].notAfter: Required value, status.controllerCertificates[3].notBefore: Required value, status.controllerCertificates[4].notAfter: Required value, status.controllerCertificates[4].notBefore: Required value, status.controllerCertificates[5].notAfter: Required value, status.controllerCertificates[5].notBefore: Required value, status.controllerCertificates[6].notAfter: Required value, status.controllerCertificates[6].notBefore: Required value, status.controllerCertificates[7].notAfter: Required value, status.controllerCertificates[7].notBefore: Required value, status.controllerCertificates[8].notAfter: Required value, status.controllerCertificates[8].notBefore: Required value, status.controllerCertificates[9].notAfter: Required value, status.controllerCertificates[9].notBefore: Required value, : Invalid value: "null": some validation rules were not checked because the object was invalid; correct the existing errors to complete validation] +20m24s Error API Availability Cluster Operator machine-config is unavailable (MachineConfigControllerFailed) diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output index ffc2c2ee40..6f6312f7b0 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output @@ -89,12 +89,58 @@ build0-gstfj-w-c-nsmn5.c.openshift-ci-build-farm.internal Completed build0-gstfj-w-c-qjbdl.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - = Update Health = -SINCE LEVEL IMPACT MESSAGE -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded | failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information -- Warning Update Speed Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable | Node is unavailable +Message: Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded + Since: - + Level: Error + Impact: Update Stalled + Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information + +Message: Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded + Since: - + Level: Error + Impact: Update Stalled + Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information + +Message: Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded + Since: - + Level: Error + Impact: Update Stalled + Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information + +Message: Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded + Since: - + Level: Error + Impact: Update Stalled + Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information + +Message: Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded + Since: - + Level: Error + Impact: Update Stalled + Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information + +Message: Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded + Since: - + Level: Error + Impact: Update Stalled + Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information + +Message: Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded + Since: - + Level: Error + Impact: Update Stalled + Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information + +Message: Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable + Since: - + Level: Warning + Impact: Update Speed + Reference: https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: Node is unavailable diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output index 7a200b483e..e8b15624a2 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output @@ -44,11 +44,11 @@ Pass along --details=nodes to see all information. = Update Health = SINCE LEVEL IMPACT MESSAGE -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded | failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information -- Warning Update Speed Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable | Node is unavailable +- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded +- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded +- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded +- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded +- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded +- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded +- Error Update Stalled Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded +- Warning Update Speed Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable diff --git a/pkg/cli/admin/upgrade/status/health.go b/pkg/cli/admin/upgrade/status/health.go index 0bd0484662..0d53c38d94 100644 --- a/pkg/cli/admin/upgrade/status/health.go +++ b/pkg/cli/admin/upgrade/status/health.go @@ -1,6 +1,7 @@ package status import ( + "fmt" "io" "sort" "strings" @@ -88,15 +89,33 @@ const ( ) type updateInsightImpact struct { - level impactLevel - impactType impactType - summary string + level impactLevel + impactType impactType + summary string + description string +} + +func (i updateInsightImpact) incomplete() bool { + return i.description == "" || i.summary == "" +} + +type updateInsightRemediation struct { + reference string +} + +func (r updateInsightRemediation) incomplete() bool { + return r.reference == "" } type updateInsight struct { - startedAt time.Time - scope updateInsightScope - impact updateInsightImpact + startedAt time.Time + scope updateInsightScope + impact updateInsightImpact + remediation updateInsightRemediation +} + +func (i updateInsight) incomplete() bool { + return i.impact.incomplete() || i.remediation.incomplete() } type updateHealthData struct { @@ -104,9 +123,17 @@ type updateHealthData struct { insights []updateInsight } -func assessUpdateInsights(insights []updateInsight, upgradingFor time.Duration, evaluatedAt time.Time) updateHealthData { +// assessUpdateInsights processes insights to be displayed and returns matching displayable data. If the displayable data are not +// worth showing in detailed mode, returns false as the second return value. +func assessUpdateInsights(insights []updateInsight, upgradingFor time.Duration, evaluatedAt time.Time) (updateHealthData, bool) { sorted := make([]updateInsight, 0, len(insights)) - sorted = append(sorted, insights...) + for _, insight := range insights { + if insight.incomplete() { + continue + } + sorted = append(sorted, insight) + } + sort.Slice(sorted, func(i, j int) bool { return sorted[i].startedAt.After(sorted[j].startedAt) }) @@ -115,20 +142,25 @@ func assessUpdateInsights(insights []updateInsight, upgradingFor time.Duration, }) if len(sorted) == 0 { - sorted = append(sorted, updateInsight{ - startedAt: evaluatedAt.Add(-upgradingFor), - impact: updateInsightImpact{ - level: infoImpactLevel, - impactType: noneImpactType, - summary: "Upgrade is proceeding well", + return updateHealthData{ + evaluatedAt: evaluatedAt, + insights: []updateInsight{ + { + startedAt: evaluatedAt.Add(-upgradingFor), + impact: updateInsightImpact{ + level: infoImpactLevel, + impactType: noneImpactType, + summary: "Upgrade is proceeding well", + }, + }, }, - }) + }, false } return updateHealthData{ evaluatedAt: evaluatedAt, insights: sorted, - } + }, true } func shortDuration(d time.Duration) string { @@ -155,17 +187,64 @@ func stringSince(health *updateHealthData, insight updateInsight) string { return shortDuration(health.evaluatedAt.Sub(insight.startedAt).Truncate(time.Second)) } -func (i *updateHealthData) Write(w io.Writer) error { +type displayItem struct { + since string + level string + impact string + message string + description string + reference string +} + +func (i *updateHealthData) Write(w io.Writer, detailed bool) error { _, _ = w.Write([]byte("= Update Health =\n")) - tabw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) - _, _ = tabw.Write([]byte("SINCE\tLEVEL\tIMPACT\tMESSAGE\n")) + displayData := make([]displayItem, 0, len(i.insights)) for _, insight := range i.insights { - _, _ = tabw.Write([]byte(stringSince(i, insight) + "\t")) - _, _ = tabw.Write([]byte(insight.impact.level.String() + "\t")) - _, _ = tabw.Write([]byte(insight.impact.impactType + "\t")) - _, _ = tabw.Write([]byte(insight.impact.summary + "\n")) + displayData = append(displayData, displayItem{ + since: stringSince(i, insight), + level: insight.impact.level.String(), + impact: string(insight.impact.impactType), + message: insight.impact.summary, + description: insight.impact.description, + reference: insight.remediation.reference, + }) + } + + if detailed { + detailedOutput(w, displayData) + } else { + tabulatedOutput(w, displayData) } - tabw.Flush() return nil } + +func detailedOutput(w io.Writer, items []displayItem) { + pad := len("Description: ") + for i, item := range items { + _, _ = w.Write([]byte(fmt.Sprintf("Message: %s\n", item.message))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Since:", item.since))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Level:", item.level))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Impact:", item.impact))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Reference:", item.reference))) + // Respect the " Description: " indentation when description has linebreaks + item.description = strings.ReplaceAll(item.description, "\n", fmt.Sprintf("\n%s, ", strings.Repeat(" ", pad+2))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Description:", item.description))) + if len(items) > i+1 { + _, _ = w.Write([]byte("\n")) + + } + } +} + +func tabulatedOutput(w io.Writer, items []displayItem) { + tabw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + _, _ = tabw.Write([]byte("SINCE\tLEVEL\tIMPACT\tMESSAGE\n")) + for _, item := range items { + _, _ = tabw.Write([]byte(item.since + "\t")) + _, _ = tabw.Write([]byte(item.level + "\t")) + _, _ = tabw.Write([]byte(item.impact + "\t")) + _, _ = tabw.Write([]byte(item.message + "\n")) + } + _ = tabw.Flush() +} diff --git a/pkg/cli/admin/upgrade/status/health_test.go b/pkg/cli/admin/upgrade/status/health_test.go index 36a04d389a..2114da8fb9 100644 --- a/pkg/cli/admin/upgrade/status/health_test.go +++ b/pkg/cli/admin/upgrade/status/health_test.go @@ -14,48 +14,58 @@ var insights = []updateInsight{ { startedAt: now, impact: updateInsightImpact{ - level: infoImpactLevel, - impactType: noneImpactType, - summary: "Something with no impact that happened right now", + level: infoImpactLevel, + impactType: noneImpactType, + summary: "Something with no impact that happened right now", + description: "This is a test", }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com"}, }, { startedAt: now.Add(-10 * time.Second), impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Something that slows the update", + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Something that slows the update", + description: "Your pathetic hardware is slowing OCP down", }, + remediation: updateInsightRemediation{reference: "https://cs.wikipedia.org/wiki/Hardware"}, }, { startedAt: now.Add(-20 * time.Second), impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: clusterCapacityImpactType, - summary: "Something that limits cluster capacity", + level: errorImpactLevel, + impactType: clusterCapacityImpactType, + summary: "Something that limits cluster capacity", + description: "Autoscaler is disabled, you should enable it", }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com/container-platform/4.14/nodes/pods/nodes-pods-autoscaling.html"}, }, { startedAt: now.Add(-5 * time.Second), impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Something that broke API and happened recently", + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Something that broke API and happened recently", + description: "Only one auth replica is available", }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com/container-platform/4.14/authentication/understanding-authentication.html"}, }, { startedAt: now.Add(-time.Hour), impact: updateInsightImpact{ - level: criticalInfoLevel, - impactType: dataLossImpactType, - summary: "KTHXBAI DATA", + level: criticalInfoLevel, + impactType: dataLossImpactType, + summary: "KTHXBAI DATA", + description: "You lost all your data, hope you have a backup", }, + remediation: updateInsightRemediation{reference: "https://en.wikipedia.org/wiki/Backup/"}, }, } func TestAssessUpdateInsights_Sorts(t *testing.T) { assessedAt := now - healthData := assessUpdateInsights(insights, 2*time.Hour, assessedAt) + healthData, allowDetailed := assessUpdateInsights(insights, 2*time.Hour, assessedAt) var messages []string for _, insights := range healthData.insights { messages = append(messages, insights.impact.summary) @@ -67,6 +77,11 @@ func TestAssessUpdateInsights_Sorts(t *testing.T) { "Something that slows the update", "Something with no impact that happened right now", } + + if !allowDetailed { + t.Error("Non-artificial insights should be allowed for detailed output") + } + if diff := cmp.Diff(expected, messages); diff != "" { t.Fatalf("Output differs from expected :\n%s", diff) } @@ -74,7 +89,7 @@ func TestAssessUpdateInsights_Sorts(t *testing.T) { func TestAssessUpdateInsights_NoInsightsCreatesAllIsWellInfo(t *testing.T) { assessedAt := now - healthData := assessUpdateInsights(nil, 2*time.Hour, assessedAt) + healthData, allowDetailed := assessUpdateInsights(nil, 2*time.Hour, assessedAt) expected := updateHealthData{ evaluatedAt: assessedAt, insights: []updateInsight{ @@ -89,26 +104,116 @@ func TestAssessUpdateInsights_NoInsightsCreatesAllIsWellInfo(t *testing.T) { }, } - if diff := cmp.Diff(expected, healthData, cmp.AllowUnexported(updateHealthData{}, updateInsight{}, updateInsightScope{}, updateInsightImpact{})); diff != "" { + if allowDetailed { + t.Error("All is well insight should not be allowed for detailed output") + } + + if diff := cmp.Diff(expected, healthData, allowUnexportedInsightStructs); diff != "" { t.Fatalf("Output differs from expected :\n%s", diff) } } -func TestUpdateHealthData_Write(t *testing.T) { - var w strings.Builder - healthData := assessUpdateInsights(insights, 2*time.Hour, now) - if err := healthData.Write(&w); err != nil { - t.Fatalf("Unexpected error: %v", err) +func TestAssessUpdateInsights_FiltersOutIncompleteInsights(t *testing.T) { + assessedAt := now + var insights = []updateInsight{ + { + startedAt: now.Add(-20 * time.Second), + impact: updateInsightImpact{ + level: errorImpactLevel, + impactType: clusterCapacityImpactType, + // empty summary + summary: "", + description: "Autoscaler is disabled, you should enable it", + }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com/container-platform/4.14/nodes/pods/nodes-pods-autoscaling.html"}, + }, + { + startedAt: now.Add(-5 * time.Second), + impact: updateInsightImpact{ + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Something that broke API and happened recently", + // empty description + description: "", + }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com/container-platform/4.14/authentication/understanding-authentication.html"}, + }, + { + startedAt: now.Add(-time.Hour), + impact: updateInsightImpact{ + level: criticalInfoLevel, + impactType: dataLossImpactType, + summary: "KTHXBAI DATA", + description: "You lost all your data, hope you have a backup", + }, + // empty reference + remediation: updateInsightRemediation{reference: ""}, + }, + } + + healthData, allowDetailed := assessUpdateInsights(insights, 2*time.Hour, assessedAt) + // All incomplete insights are filtered out, so we only get the synthetic "All is well" insight + expected := updateHealthData{ + evaluatedAt: assessedAt, + insights: []updateInsight{ + { + startedAt: assessedAt.Add(-2 * time.Hour), + impact: updateInsightImpact{ + level: infoImpactLevel, + impactType: noneImpactType, + summary: "Upgrade is proceeding well", + }, + }, + }, } - var expected = "= Update Health =\n" + - "SINCE LEVEL IMPACT MESSAGE\n" + - "1h Critical Data Loss KTHXBAI DATA\n" + - "5s Error API Availability Something that broke API and happened recently\n" + - "20s Error Cluster Capacity Something that limits cluster capacity\n" + - "10s Warning Update Speed Something that slows the update\n" + - "0s Info None Something with no impact that happened right now\n" - if diff := cmp.Diff(expected, w.String()); diff != "" { + if allowDetailed { + t.Error("All is well insight should not be allowed for detailed output") + } + + if diff := cmp.Diff(expected, healthData, cmp.AllowUnexported(updateHealthData{}, updateInsight{}, updateInsightScope{}, updateInsightImpact{}, updateInsightRemediation{})); diff != "" { t.Fatalf("Output differs from expected :\n%s", diff) } } + +func TestUpdateHealthData_Write(t *testing.T) { + testCases := []struct { + name string + detailed bool + expected string + }{ + { + name: "not detailed", + detailed: false, + expected: "= Update Health =\n" + + "SINCE LEVEL IMPACT MESSAGE\n" + + "1h Critical Data Loss KTHXBAI DATA\n" + + "5s Error API Availability Something that broke API and happened recently\n" + + "20s Error Cluster Capacity Something that limits cluster capacity\n" + + "10s Warning Update Speed Something that slows the update\n" + + "0s Info None Something with no impact that happened right now\n", + }, + { + name: "detailed", + detailed: true, + expected: "= Update Health =\n" + + "Message: KTHXBAI DATA\n Since: 1h\n Level: Critical\n Impact: Data Loss\n Reference: https://en.wikipedia.org/wiki/Backup/\n Description: You lost all your data, hope you have a backup\n\n" + + "Message: Something that broke API and happened recently\n Since: 5s\n Level: Error\n Impact: API Availability\n Reference: https://docs.openshift.com/container-platform/4.14/authentication/understanding-authentication.html\n Description: Only one auth replica is available\n\n" + + "Message: Something that limits cluster capacity\n Since: 20s\n Level: Error\n Impact: Cluster Capacity\n Reference: https://docs.openshift.com/container-platform/4.14/nodes/pods/nodes-pods-autoscaling.html\n Description: Autoscaler is disabled, you should enable it\n\n" + + "Message: Something that slows the update\n Since: 10s\n Level: Warning\n Impact: Update Speed\n Reference: https://cs.wikipedia.org/wiki/Hardware\n Description: Your pathetic hardware is slowing OCP down\n\n" + + "Message: Something with no impact that happened right now\n Since: 0s\n Level: Info\n Impact: None\n Reference: https://docs.openshift.com\n Description: This is a test\n", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var w strings.Builder + healthData, _ := assessUpdateInsights(insights, 2*time.Hour, now) + if err := healthData.Write(&w, tc.detailed); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if diff := cmp.Diff(tc.expected, w.String()); diff != "" { + t.Fatalf("Output differs from expected :\n%s", diff) + } + }) + } +} diff --git a/pkg/cli/admin/upgrade/status/status.go b/pkg/cli/admin/upgrade/status/status.go index 0da800b3e9..3221084aa4 100644 --- a/pkg/cli/admin/upgrade/status/status.go +++ b/pkg/cli/admin/upgrade/status/status.go @@ -274,8 +274,8 @@ func (o *options) Run(ctx context.Context) error { } fmt.Fprintf(o.Out, "\n") - upgradeHealth := assessUpdateInsights(updateInsights, updatingFor, now) - _ = upgradeHealth.Write(o.Out) + upgradeHealth, allowDetailed := assessUpdateInsights(updateInsights, updatingFor, now) + _ = upgradeHealth.Write(o.Out, allowDetailed && o.enabledDetailed(detailedOutputHealth)) return nil } diff --git a/pkg/cli/admin/upgrade/status/workerpool.go b/pkg/cli/admin/upgrade/status/workerpool.go index 1fc8d54aba..5661056c61 100644 --- a/pkg/cli/admin/upgrade/status/workerpool.go +++ b/pkg/cli/admin/upgrade/status/workerpool.go @@ -332,9 +332,13 @@ func nodeInsights(pool mcfgv1.MachineConfigPool, node corev1.Node, reason string resources: []scopeResource{{kind: scopeKindNode, name: node.Name}}, }, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: fmt.Sprintf("Node %s is unavailable | %s", node.Name, reason), + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: fmt.Sprintf("Node %s is unavailable", node.Name), + description: reason, + }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", }, }) } @@ -346,9 +350,13 @@ func nodeInsights(pool mcfgv1.MachineConfigPool, node corev1.Node, reason string resources: []scopeResource{{kind: scopeKindNode, name: node.Name}}, }, impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: updateStalledImpactType, - summary: fmt.Sprintf("Node %s is degraded | %s", node.Name, reason), + level: errorImpactLevel, + impactType: updateStalledImpactType, + summary: fmt.Sprintf("Node %s is degraded", node.Name), + description: reason, + }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", }, }) } @@ -423,6 +431,8 @@ func assessMachineConfigPool(pool mcfgv1.MachineConfigPool, nodes []nodeDisplayD } func machineConfigPoolInsights(poolDisplay poolDisplayData, pool mcfgv1.MachineConfigPool) (insights []updateInsight) { + // TODO: Only generate this insight if the pool has some work remaining that will not finish + // Depends on how MCO actually works: will it stop updating a node that already started e.g. draining?) if poolDisplay.NodesOverview.Excluded > 0 && pool.Spec.Paused { insights = append(insights, updateInsight{ startedAt: time.Time{}, @@ -431,9 +441,13 @@ func machineConfigPoolInsights(poolDisplay poolDisplayData, pool mcfgv1.MachineC resources: []scopeResource{{kind: scopeKindMachineConfigPool, name: pool.Name}}, }, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateStalledImpactType, - summary: fmt.Sprintf("Worker pool %s is paused | Outdated nodes in a paused pool will not be updated.", pool.Name), + level: warningImpactLevel, + impactType: updateStalledImpactType, + summary: fmt.Sprintf("Outdated nodes in a paused pool '%s' will not be updated", pool.Name), + description: "Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator.", + }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues", }, }) } diff --git a/pkg/cli/admin/upgrade/status/workerpool_test.go b/pkg/cli/admin/upgrade/status/workerpool_test.go index fcf5990eb5..0f1f0e5d2d 100644 --- a/pkg/cli/admin/upgrade/status/workerpool_test.go +++ b/pkg/cli/admin/upgrade/status/workerpool_test.go @@ -464,14 +464,18 @@ func Test_assessNodesStatus(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: updateStalledImpactType, - summary: "Node a is degraded | PDB prohibits draining", + level: errorImpactLevel, + impactType: updateStalledImpactType, + summary: "Node a is degraded", + description: "PDB prohibits draining", }, scope: updateInsightScope{ scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }, }, @@ -502,14 +506,18 @@ func Test_assessNodesStatus(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Node a is unavailable | Node is unavailable", + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Node a is unavailable", + description: "Node is unavailable", }, scope: updateInsightScope{ scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }}, } @@ -677,14 +685,18 @@ func Test_nodeInsights(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Node a is unavailable | Node is unavailable", + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Node a is unavailable", + description: "Node is unavailable", }, scope: updateInsightScope{ scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }, }, @@ -699,14 +711,18 @@ func Test_nodeInsights(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Node a is unavailable | Node is unavailable", + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Node a is unavailable", + description: "Node is unavailable", }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }, }, @@ -723,14 +739,18 @@ func Test_nodeInsights(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: updateStalledImpactType, - summary: "Node a is degraded | PDB prohibits draining", + level: errorImpactLevel, + impactType: updateStalledImpactType, + summary: "Node a is degraded", + description: "PDB prohibits draining", }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/latest/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }, }, @@ -957,14 +977,18 @@ func Test_assessMachineConfigPool(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateStalledImpactType, - summary: "Worker pool worker is paused | Outdated nodes in a paused pool will not be updated.", + level: warningImpactLevel, + impactType: updateStalledImpactType, + summary: "Outdated nodes in a paused pool 'worker' will not be updated", + description: "Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator.", }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, resources: []scopeResource{{kind: scopeKindMachineConfigPool, name: "worker"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues", + }, }, }}, } @@ -1049,14 +1073,18 @@ func Test_machineConfigPoolInsights(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateStalledImpactType, - summary: "Worker pool worker is paused | Outdated nodes in a paused pool will not be updated.", + level: warningImpactLevel, + impactType: updateStalledImpactType, + summary: "Outdated nodes in a paused pool 'worker' will not be updated", + description: "Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator.", }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, resources: []scopeResource{{kind: scopeKindMachineConfigPool, name: "worker"}}, }, + remediation: updateInsightRemediation{ + "https://docs.openshift.com/container-platform/latest/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues", + }, }, }, },