diff --git a/pkg/cli/admin/upgrade/status/controlplane.go b/pkg/cli/admin/upgrade/status/controlplane.go index 3ef4e7b994..a78ec102b5 100644 --- a/pkg/cli/admin/upgrade/status/controlplane.go +++ b/pkg/cli/admin/upgrade/status/controlplane.go @@ -48,10 +48,16 @@ func coInsights(name string, available v1.ClusterOperatorStatusCondition, degrad startedAt: available.LastTransitionTime.Time, scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: name}}}, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: apiAvailabilityImpactType, - summary: fmt.Sprintf("Cluster Operator %s is unavailable | %s: %s", name, available.Reason, strings.ReplaceAll(available.Message, "\n", ` // `)), + level: warningImpactLevel, + impactType: apiAvailabilityImpactType, + summary: fmt.Sprintf("Cluster Operator %s is unavailable (%s)", name, available.Reason), + description: available.Message, }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"}, + } + if available.Message == "" { + // Backfill the description if CO doesn't provide one + insight.impact.description = "" } if evaluated.After(available.LastTransitionTime.Time.Add(unavailableErrorThreshold)) { insight.impact.level = errorImpactLevel @@ -63,10 +69,16 @@ func coInsights(name string, available v1.ClusterOperatorStatusCondition, degrad startedAt: degraded.LastTransitionTime.Time, scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: name}}}, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: apiAvailabilityImpactType, - summary: fmt.Sprintf("Cluster Operator %s is degraded | %s: %s", name, degraded.Reason, strings.ReplaceAll(degraded.Message, "\n", ` // `)), + level: warningImpactLevel, + impactType: apiAvailabilityImpactType, + summary: fmt.Sprintf("Cluster Operator %s is degraded (%s)", name, degraded.Reason), + description: degraded.Message, }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md"}, + } + if degraded.Message == "" { + // Backfill the description if CO doesn't provide one + insight.impact.description = "" } if evaluated.After(degraded.LastTransitionTime.Time.Add(degradedErrorThreshold)) { insight.impact.level = errorImpactLevel diff --git a/pkg/cli/admin/upgrade/status/controlplane_test.go b/pkg/cli/admin/upgrade/status/controlplane_test.go index 9e4eb40044..dc5a6d4db1 100644 --- a/pkg/cli/admin/upgrade/status/controlplane_test.go +++ b/pkg/cli/admin/upgrade/status/controlplane_test.go @@ -129,7 +129,15 @@ var cvFixture = configv1.ClusterVersion{ }, } -var allowUnexportedInsightStructs = cmp.AllowUnexported(updateInsight{}, updateInsightScope{}, scopeResource{}, updateInsightImpact{}) +var allowUnexportedInsightStructs = cmp.AllowUnexported( + updateInsight{}, + updateInsightScope{}, + scopeResource{}, + updateInsightImpact{}, + updateInsightRemediation{}, + unavailableNodesInsight{}, + degradedNodesInsight{}, +) func TestAssessControlPlaneStatus_Operators(t *testing.T) { testCases := []struct { @@ -404,19 +412,23 @@ func TestCoInsights(t *testing.T) { startedAt: anchorTime.Add(-unavailableWarningThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Cluster Operator testOperator is unavailable | Broken: Operator is broken", + level: warningImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Cluster Operator testOperator is unavailable (Broken)", + description: "Operator is broken", }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"}, }, { startedAt: anchorTime.Add(-degradedWarningThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Cluster Operator testOperator is degraded | Slow: Networking is hard", + level: warningImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Cluster Operator testOperator is degraded (Slow)", + description: "Networking is hard", }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md"}, }, }, }, @@ -441,19 +453,23 @@ func TestCoInsights(t *testing.T) { startedAt: anchorTime.Add(-unavailableErrorThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Cluster Operator testOperator is unavailable | Broken: Operator is broken", + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Cluster Operator testOperator is unavailable (Broken)", + description: "Operator is broken", }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"}, }, { startedAt: anchorTime.Add(-degradedErrorThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Cluster Operator testOperator is degraded | Slow: Networking is hard", + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Cluster Operator testOperator is degraded (Slow)", + description: "Networking is hard", }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md"}, }, }, }, @@ -475,10 +491,12 @@ func TestCoInsights(t *testing.T) { startedAt: anchorTime.Add(-unavailableErrorThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: apiAvailabilityImpactType, - summary: `Cluster Operator testOperator is unavailable | Broken: Operator is broken // and message has linebreaks`, + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: `Cluster Operator testOperator is unavailable (Broken)`, + description: "Operator is broken\nand message has linebreaks", }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"}, }, }, }, diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output new file mode 100644 index 0000000000..65ac2d42de --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output @@ -0,0 +1,81 @@ +An update is in progress for 1h58m50s: Unable to apply 4.14.1: wait has exceeded 40 minutes for these operators: etcd, kube-apiserver + +Failing=True: + + Reason: ClusterOperatorsDegraded + Message: Cluster operators etcd, kube-apiserver are degraded + + += Control Plane = +Assessment: Progressing +Completion: 97% +Duration: 1h58m50s +Operator Status: 33 Total, 32 Available, 1 Progressing, 4 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Pending +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? + += Update Health = +Message: Cluster Operator kube-apiserver is degraded (NodeController_MasterNodesReady) + Since: 58m18s + Level: Error + Impact: API Availability + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Resources: + ClusterOperators: kube-apiserver + Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) + +Message: Cluster Operator kube-controller-manager is degraded (NodeController_MasterNodesReady) + Since: 58m18s + Level: Error + Impact: API Availability + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Resources: + ClusterOperators: kube-controller-manager + Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) + +Message: Cluster Operator kube-scheduler is degraded (NodeController_MasterNodesReady) + Since: 58m18s + Level: Error + Impact: API Availability + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Resources: + ClusterOperators: kube-scheduler + Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) + +Message: Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady) + Since: 58m38s + Level: Error + Impact: API Availability + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Resources: + ClusterOperators: etcd + Description: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] + , EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy + , NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) + +Message: Cluster Operator control-plane-machine-set is unavailable (UnavailableReplicas) + Since: 1h0m17s + Level: Error + Impact: API Availability + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md + Resources: + ClusterOperators: control-plane-machine-set + Description: Missing 1 available replica(s) diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output index 35aa29b7c7..e832e2ca1a 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output @@ -34,8 +34,8 @@ ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 = Update Health = SINCE LEVEL IMPACT MESSAGE -58m18s Error API Availability Cluster Operator kube-apiserver is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -58m18s Error API Availability Cluster Operator kube-controller-manager is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -58m18s Error API Availability Cluster Operator kube-scheduler is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -58m38s Error API Availability Cluster Operator etcd is degraded | EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] // EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy // NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -1h0m17s Error API Availability Cluster Operator control-plane-machine-set is unavailable | UnavailableReplicas: Missing 1 available replica(s) +58m18s Error API Availability Cluster Operator kube-apiserver is degraded (NodeController_MasterNodesReady) +58m18s Error API Availability Cluster Operator kube-controller-manager is degraded (NodeController_MasterNodesReady) +58m18s Error API Availability Cluster Operator kube-scheduler is degraded (NodeController_MasterNodesReady) +58m38s Error API Availability Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady) +1h0m17s Error API Availability Cluster Operator control-plane-machine-set is unavailable (UnavailableReplicas) diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output new file mode 100644 index 0000000000..68a4c420b7 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output @@ -0,0 +1,37 @@ +An update is in progress for 6s: Working towards 4.14.1: 139 of 859 done (16% complete), waiting on kube-scheduler + += Control Plane = +Assessment: Progressing +Completion: 12% +Duration: 6s +Operator Status: 33 Total, 33 Available, 0 Progressing, 0 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0 ? + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Excluded +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 3 Excluded, 0 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Excluded Paused 4.14.0 - +ip-10-0-4-159.us-east-2.compute.internal Excluded Paused 4.14.0 - +ip-10-0-99-40.us-east-2.compute.internal Excluded Paused 4.14.0 - + += Update Health = +Message: Outdated nodes in a paused pool 'worker' will not be updated + Since: - + Level: Warning + Impact: Update Stalled + Reference: https://docs.openshift.com/container-platform/4.15/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues + Resources: + MachineConfigPools: worker + Description: Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator. diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output index dd22bdd9e8..efa519b55d 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output @@ -28,4 +28,4 @@ ip-10-0-99-40.us-east-2.compute.internal Excluded Paused 4.14.0 - = Update Health = SINCE LEVEL IMPACT MESSAGE -- Warning Update Stalled Worker pool worker is paused | Outdated nodes in a paused pool will not be updated. +- Warning Update Stalled Outdated nodes in a paused pool 'worker' will not be updated diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output new file mode 100644 index 0000000000..b63a6ea757 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output @@ -0,0 +1,41 @@ +An update is in progress for 14m4s: Working towards 4.14.1: 734 of 859 done (85% complete), waiting on machine-config + += Control Plane = +Assessment: Progressing +Completion: 97% +Duration: 14m4s +Operator Status: 33 Total, 32 Available, 1 Progressing, 0 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-53-40.us-east-2.compute.internal Progressing Draining 4.14.0 +30m +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0 ? + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Progressing +Completion: 0% +Worker Status: 3 Total, 2 Available, 1 Progressing, 3 Outdated, 1 Draining, 0 Excluded, 0 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-4-159.us-east-2.compute.internal Progressing Draining 4.14.0 +30m +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.0 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0 ? + += Worker Pool = +Worker Pool: infra +Assessment: Progressing +Completion: 0% +Worker Status: 1 Total, 0 Available, 1 Progressing, 1 Outdated, 1 Draining, 0 Excluded, 0 Degraded + +Worker Pool Node +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-4-159-infra.us-east-2.compute.internal Progressing Draining 4.14.0 +30m + += Update Health = +SINCE LEVEL IMPACT MESSAGE +14m4s Info None Upgrade is proceeding well diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output new file mode 100644 index 0000000000..b52d104618 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output @@ -0,0 +1,31 @@ +An update is in progress for 52m56s: Working towards 4.15.0-ec.2: 357 of 1021 done (34% complete), waiting up to 40 minutes on cluster-api + += Control Plane = +Assessment: Progressing +Completion: 43% +Duration: 52m56s +Operator Status: 7 Total, 7 Available, 0 Progressing, 0 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Pending +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? +ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? + += Update Health = +SINCE LEVEL IMPACT MESSAGE +52m56s Info None Upgrade is proceeding well diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output new file mode 100644 index 0000000000..6567f43f52 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output @@ -0,0 +1,31 @@ +An update is in progress for 1m29s: Working towards 4.15.0-ec.2: 106 of 863 done (12% complete), waiting on etcd, kube-apiserver + += Control Plane = +Assessment: Progressing +Completion: 3% +Duration: 1m29s +Operator Status: 33 Total, 33 Available, 2 Progressing, 0 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.1 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.1 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.1 ? + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Pending +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.1 ? +ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.1 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.1 ? + += Update Health = +SINCE LEVEL IMPACT MESSAGE +1m29s Info None Upgrade is proceeding well diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output new file mode 100644 index 0000000000..b8fb80428f --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output @@ -0,0 +1,37 @@ +An update is in progress for 58m53s: Working towards 4.15.0-ec.2: 110 of 863 done (12% complete), waiting up to 40 minutes on etcd + += Control Plane = +Assessment: Progressing +Completion: 97% +Duration: 58m53s +Operator Status: 33 Total, 31 Available, 1 Progressing, 1 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Pending +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? + += Update Health = +Message: Cluster Operator machine-config is unavailable (MachineConfigControllerFailed) + Since: 20m24s + Level: Error + Impact: API Availability + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md + Resources: + ClusterOperators: machine-config + Description: Cluster not available for [{operator 4.14.0-rc.3}]: ControllerConfig.machineconfiguration.openshift.io "machine-config-controller" is invalid: [status.controllerCertificates[0].notAfter: Required value, status.controllerCertificates[0].notBefore: Required value, status.controllerCertificates[1].notAfter: Required value, status.controllerCertificates[1].notBefore: Required value, status.controllerCertificates[2].notAfter: Required value, status.controllerCertificates[2].notBefore: Required value, status.controllerCertificates[3].notAfter: Required value, status.controllerCertificates[3].notBefore: Required value, status.controllerCertificates[4].notAfter: Required value, status.controllerCertificates[4].notBefore: Required value, status.controllerCertificates[5].notAfter: Required value, status.controllerCertificates[5].notBefore: Required value, status.controllerCertificates[6].notAfter: Required value, status.controllerCertificates[6].notBefore: Required value, status.controllerCertificates[7].notAfter: Required value, status.controllerCertificates[7].notBefore: Required value, status.controllerCertificates[8].notAfter: Required value, status.controllerCertificates[8].notBefore: Required value, status.controllerCertificates[9].notAfter: Required value, status.controllerCertificates[9].notBefore: Required value, : Invalid value: "null": some validation rules were not checked because the object was invalid; correct the existing errors to complete validation] diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output index af5e71c194..1f241b314c 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output @@ -28,4 +28,4 @@ ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 = Update Health = SINCE LEVEL IMPACT MESSAGE -20m24s Error API Availability Cluster Operator machine-config is unavailable | MachineConfigControllerFailed: Cluster not available for [{operator 4.14.0-rc.3}]: ControllerConfig.machineconfiguration.openshift.io "machine-config-controller" is invalid: [status.controllerCertificates[0].notAfter: Required value, status.controllerCertificates[0].notBefore: Required value, status.controllerCertificates[1].notAfter: Required value, status.controllerCertificates[1].notBefore: Required value, status.controllerCertificates[2].notAfter: Required value, status.controllerCertificates[2].notBefore: Required value, status.controllerCertificates[3].notAfter: Required value, status.controllerCertificates[3].notBefore: Required value, status.controllerCertificates[4].notAfter: Required value, status.controllerCertificates[4].notBefore: Required value, status.controllerCertificates[5].notAfter: Required value, status.controllerCertificates[5].notBefore: Required value, status.controllerCertificates[6].notAfter: Required value, status.controllerCertificates[6].notBefore: Required value, status.controllerCertificates[7].notAfter: Required value, status.controllerCertificates[7].notBefore: Required value, status.controllerCertificates[8].notAfter: Required value, status.controllerCertificates[8].notBefore: Required value, status.controllerCertificates[9].notAfter: Required value, status.controllerCertificates[9].notBefore: Required value, : Invalid value: "null": some validation rules were not checked because the object was invalid; correct the existing errors to complete validation] +20m24s Error API Availability Cluster Operator machine-config is unavailable (MachineConfigControllerFailed) diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output new file mode 100644 index 0000000000..f77435b617 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output @@ -0,0 +1,108 @@ +An update is in progress for 4h3m46s: Error while reconciling 4.16.0-ec.3: the cluster operator machine-config is degraded + +Failing=True: + + Reason: ClusterOperatorDegraded + Message: Cluster operator machine-config is degraded + + += Control Plane = +Assessment: Completed +Completion: 100% +Duration: 3h30m31s +Operator Status: 36 Total, 36 Available, 0 Progressing, 1 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +build0-gstfj-m-0.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-m-1.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-m-2.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Degraded +Completion: 39% +Worker Status: 59 Total, 46 Available, 5 Progressing, 36 Outdated, 12 Draining, 0 Excluded, 7 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-c-jq5rk Unavailable Updated 4.16.0-ec.3 - Node is unavailable +build0-gstfj-ci-tests-worker-c-2kz4m Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-tests-worker-c-55hpj Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-tests-worker-c-8tmjv Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-tests-worker-c-lfz8m Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-tests-worker-c-r55wv Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-longtests-worker-c-vspm9 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-longtests-worker-d-hksc9 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-c-f6dkk Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-c-kkm72 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-c-p5df7 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-c-tgrsc Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-d-ddnxd Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-2c4vg Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-5twk4 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-65n48 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-79nrd Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-82z85 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-glhn7 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-hnxl9 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-jpzfm Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-lxqgs Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-rqch4 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-zd9c7 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-6hddk Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-kc6xt Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-r2k42 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-r9zqc Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-w2bgd Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-w-d-7rj9w.c.openshift-ci-build-farm.internal Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-builds-worker-b-65df4 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-b-dvrd2 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-b-f7rlh Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-b-m4vgr Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-b-mg8bd Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-c-2hzgm Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-c-dmjft Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-c-ng8s5 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-c-vmmfj Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-d-4pjlz Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-d-z49wm Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-tests-worker-b-d9vz2 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-tests-worker-b-fkn28 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-tests-worker-b-mc7fm Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-tests-worker-b-pqz8t Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-f7dkq.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-infra-2stfh.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-infra-mhb8g.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-infra-n9kc5.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-mwf8t.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-c-nsmn5.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-c-qjbdl.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - + += Update Health = +Message: 7 nodes in pool worker are degraded while draining: failed to drain node: after 1 hour. Please see machine-config-controller logs for more information + Since: - + Level: Error + Impact: Update Stalled + Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Resources: + Nodes: build0-gstfj-ci-prowjobs-worker-b-9lztv build0-gstfj-ci-prowjobs-worker-b-bg9f5 build0-gstfj-ci-prowjobs-worker-b-mrxwn build0-gstfj-ci-tests-worker-b-4h7pn build0-gstfj-ci-tests-worker-b-jv5bg build0-gstfj-ci-tests-worker-b-kj6gk build0-gstfj-ci-tests-worker-c-dcz9p + Description: failed to drain node: after 1 hour. Please see machine-config-controller logs for more information + +Message: Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable + Since: - + Level: Warning + Impact: Update Speed + Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Resources: + Nodes: build0-gstfj-ci-tests-worker-c-jq5rk + Description: Node is unavailable diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output index ebcd394dd6..c5ce4de1ce 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output @@ -28,27 +28,21 @@ Worker Status: 59 Total, 46 Available, 5 Progressing, 36 Outdated, 12 Draining Worker Pool Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE -build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information build0-gstfj-ci-tests-worker-c-jq5rk Unavailable Updated 4.16.0-ec.3 - Node is unavailable build0-gstfj-ci-tests-worker-c-2kz4m Progressing Draining 4.16.0-ec.2 +30m build0-gstfj-ci-tests-worker-c-55hpj Progressing Draining 4.16.0-ec.2 +30m ... Omitted additional 49 Total, 22 Completed, 46 Available, 3 Progressing, 27 Outdated, 3 Draining, 0 Excluded, and 0 Degraded nodes. -Pass along --details to see all information. +Pass along --details=nodes to see all information. = Update Health = SINCE LEVEL IMPACT MESSAGE -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded | failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information -- Warning Update Speed Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable | Node is unavailable +- Error Update Stalled 7 nodes in pool worker are degraded while draining: failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +- Warning Update Speed Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable diff --git a/pkg/cli/admin/upgrade/status/examples/not-upgrading.detailed-output b/pkg/cli/admin/upgrade/status/examples/not-upgrading.detailed-output new file mode 100644 index 0000000000..add8b8cab0 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/not-upgrading.detailed-output @@ -0,0 +1,4 @@ +The cluster version is not updating (Progressing=False). + + Reason: + Message: Cluster version is 4.14.1 diff --git a/pkg/cli/admin/upgrade/status/examples_test.go b/pkg/cli/admin/upgrade/status/examples_test.go index 7782c0d8cb..1ba2cce7bf 100644 --- a/pkg/cli/admin/upgrade/status/examples_test.go +++ b/pkg/cli/admin/upgrade/status/examples_test.go @@ -3,6 +3,7 @@ package status import ( "bytes" "context" + "fmt" "os" "path/filepath" "strings" @@ -11,9 +12,9 @@ import ( "github.com/google/go-cmp/cmp" ) -func compareWithFixture(t *testing.T, actualOut []byte, cvPath string) { +func compareWithFixture(t *testing.T, actualOut []byte, cvPath string, outputSuffix string) { t.Helper() - expectedOutPath := strings.Replace(cvPath, "-cv.yaml", ".output", 1) + expectedOutPath := strings.Replace(cvPath, "-cv.yaml", outputSuffix, 1) if update := os.Getenv("UPDATE"); update != "" { if err := os.WriteFile(expectedOutPath, actualOut, 0644); err != nil { @@ -24,10 +25,15 @@ func compareWithFixture(t *testing.T, actualOut []byte, cvPath string) { expectedOut, err := os.ReadFile(expectedOutPath) if err != nil { - t.Fatalf("Error when reading output fixture: %v", err) + if !os.IsNotExist(err) { + t.Fatalf("Error when reading output fixture: %v", err) + } else { + t.Fatalf("Output file %s does not exist. You may rerun this test with UPDATE=true to create output file with the following actual output:\n%s", expectedOutPath, actualOut) + } } + if diff := cmp.Diff(string(expectedOut), string(actualOut)); diff != "" { - t.Errorf("Output differs from expected:\n%s", diff) + t.Errorf("Output differs from expected (%s):\n%s", filepath.Base(expectedOutPath), diff) } } @@ -37,25 +43,47 @@ func TestExamples(t *testing.T) { t.Fatalf("Error when listing examples: %v", err) } + variants := []struct { + name string + detailed string + outputSuffix string + }{ + { + name: "normal output", + detailed: "none", + outputSuffix: ".output", + }, + { + name: "detailed output", + detailed: "all", + outputSuffix: ".detailed-output", + }, + } + for _, cv := range cvs { cv := cv - t.Run(cv, func(t *testing.T) { - t.Parallel() - - opts := &options{mockData: mockData{cvPath: cv}} - if err := opts.Complete(nil, nil, nil); err != nil { - t.Fatalf("Error when completing options: %v", err) - } + for _, variant := range variants { + variant := variant + t.Run(fmt.Sprintf("%s-%s", cv, variant.name), func(t *testing.T) { + t.Parallel() + opts := &options{ + mockData: mockData{cvPath: cv}, + detailedOutput: variant.detailed, + } + if err := opts.Complete(nil, nil, nil); err != nil { + t.Fatalf("Error when completing options: %v", err) + } - var stdout, stderr bytes.Buffer - opts.Out = &stdout - opts.ErrOut = &stderr + var stdout, stderr bytes.Buffer + opts.Out = &stdout + opts.ErrOut = &stderr - if err := opts.Run(context.Background()); err != nil { - t.Fatalf("Error when running: %v", err) - } + if err := opts.Run(context.Background()); err != nil { + t.Fatalf("Error when running: %v", err) + } - compareWithFixture(t, stdout.Bytes(), cv) - }) + compareWithFixture(t, stdout.Bytes(), cv, variant.outputSuffix) + }) + } } } diff --git a/pkg/cli/admin/upgrade/status/health.go b/pkg/cli/admin/upgrade/status/health.go index 0bd0484662..7ab2d1294a 100644 --- a/pkg/cli/admin/upgrade/status/health.go +++ b/pkg/cli/admin/upgrade/status/health.go @@ -1,11 +1,14 @@ package status import ( + "fmt" "io" "sort" "strings" "text/tabwriter" "time" + + "k8s.io/apimachinery/pkg/util/sets" ) type scopeType string @@ -29,6 +32,14 @@ type scopeResource struct { name string } +func (r scopeResource) namespacedName() string { + if r.namespace == "" { + return r.name + } else { + return fmt.Sprintf("%s/%s", r.namespace, r.name) + } +} + type updateInsightScope struct { scopeType scopeType resources []scopeResource @@ -88,15 +99,33 @@ const ( ) type updateInsightImpact struct { - level impactLevel - impactType impactType - summary string + level impactLevel + impactType impactType + summary string + description string +} + +func (i updateInsightImpact) incomplete() bool { + return i.description == "" || i.summary == "" +} + +type updateInsightRemediation struct { + reference string +} + +func (r updateInsightRemediation) incomplete() bool { + return r.reference == "" } type updateInsight struct { - startedAt time.Time - scope updateInsightScope - impact updateInsightImpact + startedAt time.Time + scope updateInsightScope + impact updateInsightImpact + remediation updateInsightRemediation +} + +func (i updateInsight) incomplete() bool { + return i.impact.incomplete() || i.remediation.incomplete() } type updateHealthData struct { @@ -104,9 +133,17 @@ type updateHealthData struct { insights []updateInsight } -func assessUpdateInsights(insights []updateInsight, upgradingFor time.Duration, evaluatedAt time.Time) updateHealthData { +// assessUpdateInsights processes insights to be displayed and returns matching displayable data. If the displayable data are not +// worth showing in detailed mode, returns false as the second return value. +func assessUpdateInsights(insights []updateInsight, upgradingFor time.Duration, evaluatedAt time.Time) (updateHealthData, bool) { sorted := make([]updateInsight, 0, len(insights)) - sorted = append(sorted, insights...) + for _, insight := range insights { + if insight.incomplete() { + continue + } + sorted = append(sorted, insight) + } + sort.Slice(sorted, func(i, j int) bool { return sorted[i].startedAt.After(sorted[j].startedAt) }) @@ -115,20 +152,25 @@ func assessUpdateInsights(insights []updateInsight, upgradingFor time.Duration, }) if len(sorted) == 0 { - sorted = append(sorted, updateInsight{ - startedAt: evaluatedAt.Add(-upgradingFor), - impact: updateInsightImpact{ - level: infoImpactLevel, - impactType: noneImpactType, - summary: "Upgrade is proceeding well", + return updateHealthData{ + evaluatedAt: evaluatedAt, + insights: []updateInsight{ + { + startedAt: evaluatedAt.Add(-upgradingFor), + impact: updateInsightImpact{ + level: infoImpactLevel, + impactType: noneImpactType, + summary: "Upgrade is proceeding well", + }, + }, }, - }) + }, false } return updateHealthData{ evaluatedAt: evaluatedAt, insights: sorted, - } + }, true } func shortDuration(d time.Duration) string { @@ -155,17 +197,93 @@ func stringSince(health *updateHealthData, insight updateInsight) string { return shortDuration(health.evaluatedAt.Sub(insight.startedAt).Truncate(time.Second)) } -func (i *updateHealthData) Write(w io.Writer) error { +type displayItem struct { + since string + level string + impact string + message string + description string + reference string + resources []scopeResource +} + +func (i *updateHealthData) Write(w io.Writer, detailed bool) error { _, _ = w.Write([]byte("= Update Health =\n")) - tabw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) - _, _ = tabw.Write([]byte("SINCE\tLEVEL\tIMPACT\tMESSAGE\n")) + + displayData := make([]displayItem, 0, len(i.insights)) + for _, insight := range i.insights { - _, _ = tabw.Write([]byte(stringSince(i, insight) + "\t")) - _, _ = tabw.Write([]byte(insight.impact.level.String() + "\t")) - _, _ = tabw.Write([]byte(insight.impact.impactType + "\t")) - _, _ = tabw.Write([]byte(insight.impact.summary + "\n")) + displayData = append(displayData, displayItem{ + since: stringSince(i, insight), + level: insight.impact.level.String(), + impact: string(insight.impact.impactType), + message: insight.impact.summary, + description: insight.impact.description, + reference: insight.remediation.reference, + resources: insight.scope.resources, + }) + } + if detailed { + detailedOutput(w, displayData) + } else { + tabulatedOutput(w, displayData) } - tabw.Flush() return nil } + +func detailedResourceOutput(w io.Writer, resources []scopeResource) { + if len(resources) == 0 { + return + } + _, _ = w.Write([]byte(fmt.Sprintf(" %s\n", "Resources:"))) + var pad int + byKind := make(map[allowedScopeKind][]string) + + kinds := sets.New[allowedScopeKind]() + for _, resource := range resources { + byKind[resource.kind] = append(byKind[resource.kind], resource.namespacedName()) + kinds.Insert(resource.kind) + if n := len(resource.kind); n > pad { + pad = n + } + } + + for _, kind := range sets.List(kinds) { + _, _ = w.Write([]byte(fmt.Sprintf(" %-*ss: ", pad, kind))) + sort.Strings(byKind[kind]) + _, _ = w.Write([]byte(strings.Join(byKind[kind], " ") + "\n")) + } + +} + +func detailedOutput(w io.Writer, items []displayItem) { + pad := len("Description: ") + for i, item := range items { + _, _ = w.Write([]byte(fmt.Sprintf("Message: %s\n", item.message))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Since:", item.since))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Level:", item.level))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Impact:", item.impact))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Reference:", item.reference))) + + detailedResourceOutput(w, item.resources) + // Respect the " Description: " indentation when description has linebreaks + item.description = strings.ReplaceAll(item.description, "\n", fmt.Sprintf("\n%s, ", strings.Repeat(" ", pad+2))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Description:", item.description))) + if len(items) > i+1 { + _, _ = w.Write([]byte("\n")) + } + } +} + +func tabulatedOutput(w io.Writer, items []displayItem) { + tabw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + _, _ = tabw.Write([]byte("SINCE\tLEVEL\tIMPACT\tMESSAGE\n")) + for _, item := range items { + _, _ = tabw.Write([]byte(item.since + "\t")) + _, _ = tabw.Write([]byte(item.level + "\t")) + _, _ = tabw.Write([]byte(item.impact + "\t")) + _, _ = tabw.Write([]byte(item.message + "\n")) + } + tabw.Flush() +} diff --git a/pkg/cli/admin/upgrade/status/health_test.go b/pkg/cli/admin/upgrade/status/health_test.go index 36a04d389a..ef58fd46f4 100644 --- a/pkg/cli/admin/upgrade/status/health_test.go +++ b/pkg/cli/admin/upgrade/status/health_test.go @@ -14,48 +14,58 @@ var insights = []updateInsight{ { startedAt: now, impact: updateInsightImpact{ - level: infoImpactLevel, - impactType: noneImpactType, - summary: "Something with no impact that happened right now", + level: infoImpactLevel, + impactType: noneImpactType, + summary: "Something with no impact that happened right now", + description: "This is a test", }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com"}, }, { startedAt: now.Add(-10 * time.Second), impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Something that slows the update", + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Something that slows the update", + description: "Your pathetic hardware is slowing OCP down", }, + remediation: updateInsightRemediation{reference: "https://cs.wikipedia.org/wiki/Hardware"}, }, { startedAt: now.Add(-20 * time.Second), impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: clusterCapacityImpactType, - summary: "Something that limits cluster capacity", + level: errorImpactLevel, + impactType: clusterCapacityImpactType, + summary: "Something that limits cluster capacity", + description: "Autoscaler is disabled, you should enable it", }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com/container-platform/4.14/nodes/pods/nodes-pods-autoscaling.html"}, }, { startedAt: now.Add(-5 * time.Second), impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Something that broke API and happened recently", + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Something that broke API and happened recently", + description: "Only one auth replica is available", }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com/container-platform/4.14/authentication/understanding-authentication.html"}, }, { startedAt: now.Add(-time.Hour), impact: updateInsightImpact{ - level: criticalInfoLevel, - impactType: dataLossImpactType, - summary: "KTHXBAI DATA", + level: criticalInfoLevel, + impactType: dataLossImpactType, + summary: "KTHXBAI DATA", + description: "You lost all your data, hope you have a backup", }, + remediation: updateInsightRemediation{reference: "https://en.wikipedia.org/wiki/Backup/"}, }, } func TestAssessUpdateInsights_Sorts(t *testing.T) { assessedAt := now - healthData := assessUpdateInsights(insights, 2*time.Hour, assessedAt) + healthData, allowDetailed := assessUpdateInsights(insights, 2*time.Hour, assessedAt) var messages []string for _, insights := range healthData.insights { messages = append(messages, insights.impact.summary) @@ -67,6 +77,11 @@ func TestAssessUpdateInsights_Sorts(t *testing.T) { "Something that slows the update", "Something with no impact that happened right now", } + + if !allowDetailed { + t.Error("Non-artificial insights should be allowed for detailed output") + } + if diff := cmp.Diff(expected, messages); diff != "" { t.Fatalf("Output differs from expected :\n%s", diff) } @@ -74,7 +89,7 @@ func TestAssessUpdateInsights_Sorts(t *testing.T) { func TestAssessUpdateInsights_NoInsightsCreatesAllIsWellInfo(t *testing.T) { assessedAt := now - healthData := assessUpdateInsights(nil, 2*time.Hour, assessedAt) + healthData, allowDetailed := assessUpdateInsights(nil, 2*time.Hour, assessedAt) expected := updateHealthData{ evaluatedAt: assessedAt, insights: []updateInsight{ @@ -89,26 +104,117 @@ func TestAssessUpdateInsights_NoInsightsCreatesAllIsWellInfo(t *testing.T) { }, } - if diff := cmp.Diff(expected, healthData, cmp.AllowUnexported(updateHealthData{}, updateInsight{}, updateInsightScope{}, updateInsightImpact{})); diff != "" { + if allowDetailed { + t.Error("All is well insight should not be allowed for detailed output") + } + + if diff := cmp.Diff(expected, healthData, cmp.AllowUnexported(updateHealthData{}, updateInsight{}, updateInsightScope{}, updateInsightImpact{}, updateInsightRemediation{})); diff != "" { t.Fatalf("Output differs from expected :\n%s", diff) } } -func TestUpdateHealthData_Write(t *testing.T) { - var w strings.Builder - healthData := assessUpdateInsights(insights, 2*time.Hour, now) - if err := healthData.Write(&w); err != nil { - t.Fatalf("Unexpected error: %v", err) +func TestAssessUpdateInsights_FiltersOutIncompleteInsights(t *testing.T) { + assessedAt := now + var insights = []updateInsight{ + { + startedAt: now.Add(-20 * time.Second), + impact: updateInsightImpact{ + level: errorImpactLevel, + impactType: clusterCapacityImpactType, + // empty summary + summary: "", + description: "Autoscaler is disabled, you should enable it", + }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com/container-platform/4.14/nodes/pods/nodes-pods-autoscaling.html"}, + }, + { + startedAt: now.Add(-5 * time.Second), + impact: updateInsightImpact{ + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Something that broke API and happened recently", + // empty description + description: "", + }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com/container-platform/4.14/authentication/understanding-authentication.html"}, + }, + { + startedAt: now.Add(-time.Hour), + impact: updateInsightImpact{ + level: criticalInfoLevel, + impactType: dataLossImpactType, + summary: "KTHXBAI DATA", + description: "You lost all your data, hope you have a backup", + }, + // empty reference + remediation: updateInsightRemediation{reference: ""}, + }, + } + + healthData, allowDetailed := assessUpdateInsights(insights, 2*time.Hour, assessedAt) + // All incomplete insights are filtered out, so we only get the synthetic "All is well" insight + expected := updateHealthData{ + evaluatedAt: assessedAt, + insights: []updateInsight{ + { + startedAt: assessedAt.Add(-2 * time.Hour), + impact: updateInsightImpact{ + level: infoImpactLevel, + impactType: noneImpactType, + summary: "Upgrade is proceeding well", + }, + }, + }, + } + + if allowDetailed { + t.Error("All is well insight should not be allowed for detailed output") } - var expected = "= Update Health =\n" + - "SINCE LEVEL IMPACT MESSAGE\n" + - "1h Critical Data Loss KTHXBAI DATA\n" + - "5s Error API Availability Something that broke API and happened recently\n" + - "20s Error Cluster Capacity Something that limits cluster capacity\n" + - "10s Warning Update Speed Something that slows the update\n" + - "0s Info None Something with no impact that happened right now\n" - - if diff := cmp.Diff(expected, w.String()); diff != "" { + + if diff := cmp.Diff(expected, healthData, cmp.AllowUnexported(updateHealthData{}, updateInsight{}, updateInsightScope{}, updateInsightImpact{}, updateInsightRemediation{})); diff != "" { t.Fatalf("Output differs from expected :\n%s", diff) } } + +func TestUpdateHealthData_Write(t *testing.T) { + testCases := []struct { + name string + detailed bool + expected string + }{ + { + name: "not detailed", + detailed: false, + expected: "= Update Health =\n" + + "SINCE LEVEL IMPACT MESSAGE\n" + + "1h Critical Data Loss KTHXBAI DATA\n" + + "5s Error API Availability Something that broke API and happened recently\n" + + "20s Error Cluster Capacity Something that limits cluster capacity\n" + + "10s Warning Update Speed Something that slows the update\n" + + "0s Info None Something with no impact that happened right now\n", + }, + { + name: "detailed", + detailed: true, + expected: "= Update Health =\n" + + "Message: KTHXBAI DATA\n Since: 1h\n Level: Critical\n Impact: Data Loss\n Reference: https://en.wikipedia.org/wiki/Backup/\n Description: You lost all your data, hope you have a backup\n\n" + + "Message: Something that broke API and happened recently\n Since: 5s\n Level: Error\n Impact: API Availability\n Reference: https://docs.openshift.com/container-platform/4.14/authentication/understanding-authentication.html\n Description: Only one auth replica is available\n\n" + + "Message: Something that limits cluster capacity\n Since: 20s\n Level: Error\n Impact: Cluster Capacity\n Reference: https://docs.openshift.com/container-platform/4.14/nodes/pods/nodes-pods-autoscaling.html\n Description: Autoscaler is disabled, you should enable it\n\n" + + "Message: Something that slows the update\n Since: 10s\n Level: Warning\n Impact: Update Speed\n Reference: https://cs.wikipedia.org/wiki/Hardware\n Description: Your pathetic hardware is slowing OCP down\n\n" + + "Message: Something with no impact that happened right now\n Since: 0s\n Level: Info\n Impact: None\n Reference: https://docs.openshift.com\n Description: This is a test\n", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var w strings.Builder + healthData, _ := assessUpdateInsights(insights, 2*time.Hour, now) + if err := healthData.Write(&w, tc.detailed); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if diff := cmp.Diff(tc.expected, w.String()); diff != "" { + t.Fatalf("Output differs from expected :\n%s", diff) + } + }) + } + +} diff --git a/pkg/cli/admin/upgrade/status/status.go b/pkg/cli/admin/upgrade/status/status.go index c50e533f79..8fa5a0aaca 100644 --- a/pkg/cli/admin/upgrade/status/status.go +++ b/pkg/cli/admin/upgrade/status/status.go @@ -11,6 +11,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/cli-runtime/pkg/genericiooptions" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" kcmdutil "k8s.io/kubectl/pkg/cmd/util" @@ -34,6 +35,23 @@ func newOptions(streams genericiooptions.IOStreams) *options { } } +const ( + detailedOutputNone = "none" + detailedOutputNodes = "nodes" + detailedOutputHealth = "health" + detailedOutputAll = "all" +) + +func detailedHealthEnabled(detailedOutput string) bool { + return detailedOutput == detailedOutputHealth || detailedOutput == detailedOutputAll +} + +func detailedNodesEnabled(detailedOutput string) bool { + return detailedOutput == detailedOutputNodes || detailedOutput == detailedOutputAll +} + +var allDetailedOutputValues = sets.New[string](detailedOutputNone, detailedOutputNodes, detailedOutputHealth, detailedOutputAll) + func New(f kcmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Command { o := newOptions(streams) cmd := &cobra.Command{ @@ -49,6 +67,7 @@ func New(f kcmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Command // TODO: We can remove these flags once the idea about `oc adm upgrade status` stabilizes and the command // is promoted out of the OC_ENABLE_CMD_UPGRADE_STATUS feature gate flags.StringVar(&o.mockData.cvPath, "mock-clusterversion", "", "Path to a YAML ClusterVersion object to use for testing (will be removed later). Files in the same directory with the same name and suffixes -co.yaml, -mcp.yaml, -mc.yaml, and -node.yaml are required.") + flags.StringVar(&o.detailedOutput, "details", detailedOutputNone, fmt.Sprintf("Show more detailed output in some sections. One of %s", strings.Join(sets.List(allDetailedOutputValues), ", "))) return cmd } @@ -61,6 +80,8 @@ type options struct { ConfigClient configv1client.Interface CoreClient corev1client.CoreV1Interface MachineConfigClient machineconfigv1client.Interface + + detailedOutput string } func (o *options) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { @@ -68,6 +89,10 @@ func (o *options) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string return kcmdutil.UsageErrorf(cmd, "positional arguments given") } + if !allDetailedOutputValues.Has(o.detailedOutput) { + return kcmdutil.UsageErrorf(cmd, "invalid value for --details: %s (must be one of: %s)", o.detailedOutput, strings.Join(allDetailedOutputValues.UnsortedList(), ", ")) + } + cvSuffix := "-cv.yaml" if o.mockData.cvPath != "" { o.mockData.operatorsPath = strings.Replace(o.mockData.cvPath, cvSuffix, "-co.yaml", 1) @@ -244,18 +269,18 @@ func (o *options) Run(ctx context.Context) error { updateInsights = append(updateInsights, insights...) fmt.Fprintf(o.Out, "\n") _ = controlPlaneStatusData.Write(o.Out) - controlPlanePoolStatusData.WriteNodes(o.Out) + controlPlanePoolStatusData.WriteNodes(o.Out, detailedNodesEnabled(o.detailedOutput)) fmt.Fprintf(o.Out, "\n= Worker Upgrade =\n") for _, pool := range workerPoolsStatusData { fmt.Fprintf(o.Out, "\n") _ = pool.WritePool(o.Out) - pool.WriteNodes(o.Out) + pool.WriteNodes(o.Out, detailedNodesEnabled(o.detailedOutput)) } fmt.Fprintf(o.Out, "\n") - upgradeHealth := assessUpdateInsights(updateInsights, updatingFor, now) - _ = upgradeHealth.Write(o.Out) + upgradeHealth, allowDetailed := assessUpdateInsights(updateInsights, updatingFor, now) + _ = upgradeHealth.Write(o.Out, detailedHealthEnabled(o.detailedOutput) && allowDetailed) return nil } diff --git a/pkg/cli/admin/upgrade/status/workerpool.go b/pkg/cli/admin/upgrade/status/workerpool.go index d61d26efad..6b8632fa1e 100644 --- a/pkg/cli/admin/upgrade/status/workerpool.go +++ b/pkg/cli/admin/upgrade/status/workerpool.go @@ -1,9 +1,11 @@ package status import ( + "cmp" "context" "fmt" "io" + "slices" "sort" "strings" "text/tabwriter" @@ -13,6 +15,7 @@ import ( corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" configv1 "github.com/openshift/api/config/v1" mcfgv1 "github.com/openshift/api/machineconfiguration/v1" @@ -135,9 +138,112 @@ func selectNodesFromPool(pool mcfgv1.MachineConfigPool, allNodes []corev1.Node) return res, nil } +type multipleNodeInsight interface { + merge(insight multipleNodeInsight) bool + render() updateInsight +} + +type unavailableNodesInsight struct { + // TODO(muller): Unwrap this + insight updateInsight +} + +func (a *unavailableNodesInsight) merge(_ multipleNodeInsight) bool { + return false +} + +func (a *unavailableNodesInsight) render() updateInsight { + return a.insight +} + +type degradedNodesInsight struct { + pool string + phase nodePhase + startedAt time.Time + scope updateInsightScope + description string +} + +func (d *degradedNodesInsight) merge(insight multipleNodeInsight) bool { + other, ok := insight.(*degradedNodesInsight) + if !ok { + return false + } + + if d.scope.scopeType != other.scope.scopeType { + return false + } + + myDesc := d.description + if len(d.scope.resources) == 1 { + nodeName := d.scope.resources[0].name + myDesc = strings.Replace(myDesc, nodeName, ellipsize(nodeName, 5), -1) + } + otherDesc := other.description + if len(other.scope.resources) == 1 { + nodeName := other.scope.resources[0].name + otherDesc = strings.Replace(otherDesc, nodeName, ellipsize(nodeName, 5), -1) + } + + if myDesc != otherDesc { + return false + } + d.description = myDesc + d.scope.resources = append(d.scope.resources, other.scope.resources...) + + if other.startedAt.Before(d.startedAt) { + d.startedAt = other.startedAt + } + + return true +} + +func (d *degradedNodesInsight) render() updateInsight { + // Deduplicate and sort resources + resources := sets.New[scopeResource](d.scope.resources...) + d.scope.resources = resources.UnsortedList() + slices.SortFunc(d.scope.resources, func(a, b scopeResource) int { + if n := cmp.Compare(a.kind, b.kind); n != 0 { + return n + } + if n := cmp.Compare(a.namespace, b.namespace); n != 0 { + return n + } + return cmp.Compare(a.name, b.name) + }) + + var node uint + for _, resource := range d.scope.resources { + if resource.kind == scopeKindNode { + node += 1 + } + } + + var summary string + if nodes := len(d.scope.resources); nodes > 1 { + summary = fmt.Sprintf("%d nodes in pool %s are degraded while %s: %s", nodes, d.pool, strings.ToLower(d.phase.String()), d.description) + } else { + summary = fmt.Sprintf("Node %s in pool %s is degraded while %s: %s", d.scope.resources[0].name, d.pool, strings.ToLower(d.phase.String()), d.description) + } + + return updateInsight{ + startedAt: d.startedAt, + scope: d.scope, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, + impact: updateInsightImpact{ + level: errorImpactLevel, + impactType: updateStalledImpactType, + summary: summary, + description: d.description, + }, + } +} + func assessNodesStatus(cv *configv1.ClusterVersion, pool mcfgv1.MachineConfigPool, nodes []corev1.Node, machineConfigs []mcfgv1.MachineConfig) ([]nodeDisplayData, []updateInsight) { var nodesStatusData []nodeDisplayData - var insights []updateInsight + var multipleNodeInsights []multipleNodeInsight for _, node := range nodes { currentVersion := getOpenShiftVersionOfMachineConfig(machineConfigs, node.Annotations[mco.CurrentMachineConfigAnnotationKey]) desiredVersion := getOpenShiftVersionOfMachineConfig(machineConfigs, node.Annotations[mco.DesiredMachineConfigAnnotationKey]) @@ -191,14 +297,27 @@ func assessNodesStatus(cv *configv1.ClusterVersion, pool mcfgv1.MachineConfigPoo } } - insights = append(insights, nodeInsights(pool, node, message, isUnavailable, isUpdating, isDegraded)...) + singleNodeInsights := nodeInsights(pool, node, message, phase, isUnavailable, isUpdating, isDegraded) + for si := range singleNodeInsights { + merged := false + insight := singleNodeInsights[si] + for mi := range multipleNodeInsights { + if ok := multipleNodeInsights[mi].merge(insight); ok { + merged = true + break + } + } + if !merged { + multipleNodeInsights = append(multipleNodeInsights, insight) + } + } nodesStatusData = append(nodesStatusData, nodeDisplayData{ Name: node.Name, Assessment: assessment, Estimate: estimate, Phase: phase, - Message: message, + Message: strings.Replace(message, node.Name, ellipsize(node.Name, 10), -1), Version: currentVersion, isUnavailable: isUnavailable, isDegraded: isDegraded, @@ -217,9 +336,23 @@ func assessNodesStatus(cv *configv1.ClusterVersion, pool mcfgv1.MachineConfigPoo return nodesStatusData[i].Assessment < nodesStatusData[j].Assessment }) + var insights []updateInsight + if len(multipleNodeInsights) > 0 { + insights = make([]updateInsight, 0, len(multipleNodeInsights)) + for i := range multipleNodeInsights { + insights = append(insights, multipleNodeInsights[i].render()) + } + } return nodesStatusData, insights } +func ellipsize(name string, aboveLen int) string { + if len(name) <= aboveLen { + return name + } + return "" +} + func getOpenShiftVersionOfMachineConfig(machineConfigs []mcfgv1.MachineConfig, name string) string { for _, mc := range machineConfigs { if mc.Name == name { @@ -318,38 +451,42 @@ func mcdUpdatingStateToPhase(state string) nodePhase { } } -func nodeInsights(pool mcfgv1.MachineConfigPool, node corev1.Node, reason string, isUnavailable, isUpdating, isDegraded bool) []updateInsight { - var insights []updateInsight +func nodeInsights(pool mcfgv1.MachineConfigPool, node corev1.Node, reason string, phase nodePhase, isUnavailable, isUpdating, isDegraded bool) []multipleNodeInsight { + var insights []multipleNodeInsight scope := scopeTypeWorkerPool if pool.Name == "master" { scope = scopeTypeControlPlane } if isUnavailable && !isUpdating { - insights = append(insights, updateInsight{ - startedAt: time.Time{}, - scope: updateInsightScope{ - scopeType: scope, - resources: []scopeResource{{kind: scopeKindNode, name: node.Name}}, - }, - impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: fmt.Sprintf("Node %s is unavailable | %s", node.Name, reason), + insights = append(insights, &unavailableNodesInsight{ + insight: updateInsight{ + startedAt: time.Time{}, + scope: updateInsightScope{ + scopeType: scope, + resources: []scopeResource{{kind: scopeKindNode, name: node.Name}}, + }, + impact: updateInsightImpact{ + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: fmt.Sprintf("Node %s is unavailable", node.Name), + description: reason, + }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }) } if isDegraded { - insights = append(insights, updateInsight{ + insights = append(insights, °radedNodesInsight{ + pool: pool.Name, + phase: phase, startedAt: time.Time{}, scope: updateInsightScope{ scopeType: scope, resources: []scopeResource{{kind: scopeKindNode, name: node.Name}}, }, - impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: updateStalledImpactType, - summary: fmt.Sprintf("Node %s is degraded | %s", node.Name, reason), - }, + description: reason, }) } return insights @@ -423,6 +560,8 @@ func assessMachineConfigPool(pool mcfgv1.MachineConfigPool, nodes []nodeDisplayD } func machineConfigPoolInsights(poolDisplay poolDisplayData, pool mcfgv1.MachineConfigPool) (insights []updateInsight) { + // TODO: Only generate this insight if the pool has some work remaining that will not finish + // Depends on how MCO actually works: will it stop updating a node that already started e.g. draining?) if poolDisplay.NodesOverview.Excluded > 0 && pool.Spec.Paused { insights = append(insights, updateInsight{ startedAt: time.Time{}, @@ -431,9 +570,13 @@ func machineConfigPoolInsights(poolDisplay poolDisplayData, pool mcfgv1.MachineC resources: []scopeResource{{kind: scopeKindMachineConfigPool, name: pool.Name}}, }, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateStalledImpactType, - summary: fmt.Sprintf("Worker pool %s is paused | Outdated nodes in a paused pool will not be updated.", pool.Name), + level: warningImpactLevel, + impactType: updateStalledImpactType, + summary: fmt.Sprintf("Outdated nodes in a paused pool '%s' will not be updated", pool.Name), + description: "Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator.", + }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues", }, }) } @@ -453,7 +596,7 @@ Completion: {{ printf "%.0f" .Completion }}% Worker Status: {{ .NodesOverview.Total }} Total, {{ .NodesOverview.Available }} Available, {{ .NodesOverview.Progressing }} Progressing, {{ .NodesOverview.Outdated }} Outdated, {{ .NodesOverview.Draining }} Draining, {{ .NodesOverview.Excluded }} Excluded, {{ .NodesOverview.Degraded }} Degraded ` -func (pool *poolDisplayData) WriteNodes(w io.Writer) { +func (pool *poolDisplayData) WriteNodes(w io.Writer, detailed bool) { if pool.Name == mco.MachineConfigPoolMaster { fmt.Fprintf(w, "\nControl Plane Node") } else { @@ -467,7 +610,7 @@ func (pool *poolDisplayData) WriteNodes(w io.Writer) { _, _ = tabw.Write([]byte("\nNAME\tASSESSMENT\tPHASE\tVERSION\tEST\tMESSAGE\n")) var total, completed, available, progressing, outdated, draining, excluded int for i, node := range pool.Nodes { - if i >= 10 { + if i >= 10 && !detailed { // Limit displaying too many nodes // Display nodes in undesired states regardless their count if !node.isDegraded && (!node.isUnavailable || node.isUpdating) { @@ -502,6 +645,6 @@ func (pool *poolDisplayData) WriteNodes(w io.Writer) { } tabw.Flush() if total > 0 { - fmt.Fprintf(w, "...\nOmitted additional %d Total, %d Completed, %d Available, %d Progressing, %d Outdated, %d Draining, %d Excluded, and 0 Degraded nodes.\nPass along --details to see all information.\n", total, completed, available, progressing, outdated, draining, excluded) + fmt.Fprintf(w, "...\nOmitted additional %d Total, %d Completed, %d Available, %d Progressing, %d Outdated, %d Draining, %d Excluded, and 0 Degraded nodes.\nPass along --details=nodes to see all information.\n", total, completed, available, progressing, outdated, draining, excluded) } } diff --git a/pkg/cli/admin/upgrade/status/workerpool_test.go b/pkg/cli/admin/upgrade/status/workerpool_test.go index fcf5990eb5..861760d9ad 100644 --- a/pkg/cli/admin/upgrade/status/workerpool_test.go +++ b/pkg/cli/admin/upgrade/status/workerpool_test.go @@ -464,14 +464,18 @@ func Test_assessNodesStatus(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: updateStalledImpactType, - summary: "Node a is degraded | PDB prohibits draining", + level: errorImpactLevel, + impactType: updateStalledImpactType, + summary: "Node a in pool master is degraded while draining: PDB prohibits draining", + description: "PDB prohibits draining", }, scope: updateInsightScope{ scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }, }, @@ -502,14 +506,18 @@ func Test_assessNodesStatus(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Node a is unavailable | Node is unavailable", + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Node a is unavailable", + description: "Node is unavailable", }, scope: updateInsightScope{ scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }}, } @@ -649,6 +657,7 @@ func Test_nodeInsights(t *testing.T) { pool mcfgv1.MachineConfigPool node corev1.Node reason string + phase nodePhase isUnavailable bool isUpdating bool isDegraded bool @@ -656,7 +665,7 @@ func Test_nodeInsights(t *testing.T) { testCases := []struct { name string args args - expectedUpdateInsight []updateInsight + expectedUpdateInsight []multipleNodeInsight }{ { name: "node is updated - all is well", @@ -672,18 +681,25 @@ func Test_nodeInsights(t *testing.T) { pool: mcpMaster, node: node("a").updated(mcNew.Name, mcNew.Name).unavailable().node, reason: "Node is unavailable", + phase: phaseStateUpdated, isUnavailable: true, }, - expectedUpdateInsight: []updateInsight{ - { - impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Node a is unavailable | Node is unavailable", - }, - scope: updateInsightScope{ - scopeType: scopeTypeControlPlane, - resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, + expectedUpdateInsight: []multipleNodeInsight{ + &unavailableNodesInsight{ + updateInsight{ + impact: updateInsightImpact{ + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Node a is unavailable", + description: "Node is unavailable", + }, + scope: updateInsightScope{ + scopeType: scopeTypeControlPlane, + resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, + }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }, }, @@ -696,16 +712,22 @@ func Test_nodeInsights(t *testing.T) { reason: "Node is unavailable", isUnavailable: true, }, - expectedUpdateInsight: []updateInsight{ - { - impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Node a is unavailable | Node is unavailable", - }, - scope: updateInsightScope{ - scopeType: scopeTypeWorkerPool, - resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, + expectedUpdateInsight: []multipleNodeInsight{ + &unavailableNodesInsight{ + updateInsight{ + impact: updateInsightImpact{ + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Node a is unavailable", + description: "Node is unavailable", + }, + scope: updateInsightScope{ + scopeType: scopeTypeWorkerPool, + resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, + }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }, }, @@ -716,28 +738,27 @@ func Test_nodeInsights(t *testing.T) { pool: mcpWorker, node: node("a").degraded_draining(mcOld.Name, mcNew.Name, "PDB prohibits draining").node, reason: "PDB prohibits draining", + phase: phaseStateDraining, isUnavailable: true, isUpdating: true, isDegraded: true, }, - expectedUpdateInsight: []updateInsight{ - { - impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: updateStalledImpactType, - summary: "Node a is degraded | PDB prohibits draining", - }, + expectedUpdateInsight: []multipleNodeInsight{ + °radedNodesInsight{ + pool: mcpWorker.Name, + phase: phaseStateDraining, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, + description: "PDB prohibits draining", }, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - updateInsights := nodeInsights(tc.args.pool, tc.args.node, tc.args.reason, tc.args.isUnavailable, tc.args.isUpdating, tc.args.isDegraded) + updateInsights := nodeInsights(tc.args.pool, tc.args.node, tc.args.reason, tc.args.phase, tc.args.isUnavailable, tc.args.isUpdating, tc.args.isDegraded) if diff := cmp.Diff(tc.expectedUpdateInsight, updateInsights, allowUnexportedInsightStructs); diff != "" { t.Errorf("updateInsight differ from expected:\n%s", diff) } @@ -957,14 +978,18 @@ func Test_assessMachineConfigPool(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateStalledImpactType, - summary: "Worker pool worker is paused | Outdated nodes in a paused pool will not be updated.", + level: warningImpactLevel, + impactType: updateStalledImpactType, + summary: "Outdated nodes in a paused pool 'worker' will not be updated", + description: "Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator.", }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, resources: []scopeResource{{kind: scopeKindMachineConfigPool, name: "worker"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues", + }, }, }}, } @@ -1049,14 +1074,18 @@ func Test_machineConfigPoolInsights(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateStalledImpactType, - summary: "Worker pool worker is paused | Outdated nodes in a paused pool will not be updated.", + level: warningImpactLevel, + impactType: updateStalledImpactType, + summary: "Outdated nodes in a paused pool 'worker' will not be updated", + description: "Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator.", }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, resources: []scopeResource{{kind: scopeKindMachineConfigPool, name: "worker"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues", + }, }, }, },