Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 18 additions & 6 deletions pkg/cli/admin/upgrade/status/controlplane.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,16 @@ func coInsights(name string, available v1.ClusterOperatorStatusCondition, degrad
startedAt: available.LastTransitionTime.Time,
scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: name}}},
impact: updateInsightImpact{
level: warningImpactLevel,
impactType: apiAvailabilityImpactType,
summary: fmt.Sprintf("Cluster Operator %s is unavailable | %s: %s", name, available.Reason, strings.ReplaceAll(available.Message, "\n", ` // `)),
level: warningImpactLevel,
impactType: apiAvailabilityImpactType,
summary: fmt.Sprintf("Cluster Operator %s is unavailable (%s)", name, available.Reason),
description: available.Message,
},
remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"},
}
if available.Message == "" {
// Backfill the description if CO doesn't provide one
insight.impact.description = "<no message>"
}
if evaluated.After(available.LastTransitionTime.Time.Add(unavailableErrorThreshold)) {
insight.impact.level = errorImpactLevel
Expand All @@ -63,10 +69,16 @@ func coInsights(name string, available v1.ClusterOperatorStatusCondition, degrad
startedAt: degraded.LastTransitionTime.Time,
scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: name}}},
impact: updateInsightImpact{
level: warningImpactLevel,
impactType: apiAvailabilityImpactType,
summary: fmt.Sprintf("Cluster Operator %s is degraded | %s: %s", name, degraded.Reason, strings.ReplaceAll(degraded.Message, "\n", ` // `)),
level: warningImpactLevel,
impactType: apiAvailabilityImpactType,
summary: fmt.Sprintf("Cluster Operator %s is degraded (%s)", name, degraded.Reason),
description: degraded.Message,
},
remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md"},
}
if degraded.Message == "" {
// Backfill the description if CO doesn't provide one
insight.impact.description = "<no message>"
}
if evaluated.After(degraded.LastTransitionTime.Time.Add(degradedErrorThreshold)) {
insight.impact.level = errorImpactLevel
Expand Down
50 changes: 34 additions & 16 deletions pkg/cli/admin/upgrade/status/controlplane_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,15 @@ var cvFixture = configv1.ClusterVersion{
},
}

var allowUnexportedInsightStructs = cmp.AllowUnexported(updateInsight{}, updateInsightScope{}, scopeResource{}, updateInsightImpact{})
var allowUnexportedInsightStructs = cmp.AllowUnexported(
updateInsight{},
updateInsightScope{},
scopeResource{},
updateInsightImpact{},
updateInsightRemediation{},
unavailableNodesInsight{},
degradedNodesInsight{},
)

func TestAssessControlPlaneStatus_Operators(t *testing.T) {
testCases := []struct {
Expand Down Expand Up @@ -404,19 +412,23 @@ func TestCoInsights(t *testing.T) {
startedAt: anchorTime.Add(-unavailableWarningThreshold).Add(-time.Second),
scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}},
impact: updateInsightImpact{
level: warningImpactLevel,
impactType: apiAvailabilityImpactType,
summary: "Cluster Operator testOperator is unavailable | Broken: Operator is broken",
level: warningImpactLevel,
impactType: apiAvailabilityImpactType,
summary: "Cluster Operator testOperator is unavailable (Broken)",
description: "Operator is broken",
},
remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"},
},
{
startedAt: anchorTime.Add(-degradedWarningThreshold).Add(-time.Second),
scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}},
impact: updateInsightImpact{
level: warningImpactLevel,
impactType: apiAvailabilityImpactType,
summary: "Cluster Operator testOperator is degraded | Slow: Networking is hard",
level: warningImpactLevel,
impactType: apiAvailabilityImpactType,
summary: "Cluster Operator testOperator is degraded (Slow)",
description: "Networking is hard",
},
remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md"},
},
},
},
Expand All @@ -441,19 +453,23 @@ func TestCoInsights(t *testing.T) {
startedAt: anchorTime.Add(-unavailableErrorThreshold).Add(-time.Second),
scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}},
impact: updateInsightImpact{
level: errorImpactLevel,
impactType: apiAvailabilityImpactType,
summary: "Cluster Operator testOperator is unavailable | Broken: Operator is broken",
level: errorImpactLevel,
impactType: apiAvailabilityImpactType,
summary: "Cluster Operator testOperator is unavailable (Broken)",
description: "Operator is broken",
},
remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"},
},
{
startedAt: anchorTime.Add(-degradedErrorThreshold).Add(-time.Second),
scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}},
impact: updateInsightImpact{
level: errorImpactLevel,
impactType: apiAvailabilityImpactType,
summary: "Cluster Operator testOperator is degraded | Slow: Networking is hard",
level: errorImpactLevel,
impactType: apiAvailabilityImpactType,
summary: "Cluster Operator testOperator is degraded (Slow)",
description: "Networking is hard",
},
remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md"},
},
},
},
Expand All @@ -475,10 +491,12 @@ func TestCoInsights(t *testing.T) {
startedAt: anchorTime.Add(-unavailableErrorThreshold).Add(-time.Second),
scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}},
impact: updateInsightImpact{
level: errorImpactLevel,
impactType: apiAvailabilityImpactType,
summary: `Cluster Operator testOperator is unavailable | Broken: Operator is broken // and message has linebreaks`,
level: errorImpactLevel,
impactType: apiAvailabilityImpactType,
summary: `Cluster Operator testOperator is unavailable (Broken)`,
description: "Operator is broken\nand message has linebreaks",

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest to update the test name to reflect the change.

Something like: "insights flatten linebreaks in messages" -> "insights do not flatten linebreaks in messages"

},
remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"},
},
},
},
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
An update is in progress for 1h58m50s: Unable to apply 4.14.1: wait has exceeded 40 minutes for these operators: etcd, kube-apiserver

Failing=True:

Reason: ClusterOperatorsDegraded
Message: Cluster operators etcd, kube-apiserver are degraded


= Control Plane =
Assessment: Progressing
Completion: 97%
Duration: 1h58m50s
Operator Status: 33 Total, 32 Available, 1 Progressing, 4 Degraded

Control Plane Nodes
NAME ASSESSMENT PHASE VERSION EST MESSAGE
ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ?
ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ?
ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ?

= Worker Upgrade =

= Worker Pool =
Worker Pool: worker
Assessment: Pending
Completion: 0%
Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded

Worker Pool Nodes
NAME ASSESSMENT PHASE VERSION EST MESSAGE
ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ?
ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ?
ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ?

= Update Health =
Message: Cluster Operator kube-apiserver is degraded (NodeController_MasterNodesReady)
Since: 58m18s
Level: Error
Impact: API Availability
Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md
Resources:
ClusterOperators: kube-apiserver
Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?)

Message: Cluster Operator kube-controller-manager is degraded (NodeController_MasterNodesReady)
Since: 58m18s
Level: Error
Impact: API Availability
Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md
Resources:
ClusterOperators: kube-controller-manager
Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?)

Message: Cluster Operator kube-scheduler is degraded (NodeController_MasterNodesReady)
Since: 58m18s
Level: Error
Impact: API Availability
Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md
Resources:
ClusterOperators: kube-scheduler
Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?)

Message: Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady)
Since: 58m38s
Level: Error
Impact: API Availability
Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md
Resources:
ClusterOperators: etcd
Description: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:<nil>} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:<nil>} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}]
, EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy
, NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?)

Message: Cluster Operator control-plane-machine-set is unavailable (UnavailableReplicas)
Since: 1h0m17s
Level: Error
Impact: API Availability
Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md
Resources:
ClusterOperators: control-plane-machine-set
Description: Missing 1 available replica(s)
10 changes: 5 additions & 5 deletions pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3

= Update Health =
SINCE LEVEL IMPACT MESSAGE
58m18s Error API Availability Cluster Operator kube-apiserver is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?)
58m18s Error API Availability Cluster Operator kube-controller-manager is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?)
58m18s Error API Availability Cluster Operator kube-scheduler is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?)
58m38s Error API Availability Cluster Operator etcd is degraded | EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:<nil>} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:<nil>} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] // EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy // NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?)
1h0m17s Error API Availability Cluster Operator control-plane-machine-set is unavailable | UnavailableReplicas: Missing 1 available replica(s)
58m18s Error API Availability Cluster Operator kube-apiserver is degraded (NodeController_MasterNodesReady)
58m18s Error API Availability Cluster Operator kube-controller-manager is degraded (NodeController_MasterNodesReady)
58m18s Error API Availability Cluster Operator kube-scheduler is degraded (NodeController_MasterNodesReady)
58m38s Error API Availability Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady)
1h0m17s Error API Availability Cluster Operator control-plane-machine-set is unavailable (UnavailableReplicas)
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
An update is in progress for 6s: Working towards 4.14.1: 139 of 859 done (16% complete), waiting on kube-scheduler

= Control Plane =
Assessment: Progressing
Completion: 12%
Duration: 6s
Operator Status: 33 Total, 33 Available, 0 Progressing, 0 Degraded

Control Plane Nodes
NAME ASSESSMENT PHASE VERSION EST MESSAGE
ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0 ?
ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0 ?
ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0 ?

= Worker Upgrade =

= Worker Pool =
Worker Pool: worker
Assessment: Excluded
Completion: 0%
Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 3 Excluded, 0 Degraded

Worker Pool Nodes
NAME ASSESSMENT PHASE VERSION EST MESSAGE
ip-10-0-20-162.us-east-2.compute.internal Excluded Paused 4.14.0 -
ip-10-0-4-159.us-east-2.compute.internal Excluded Paused 4.14.0 -
ip-10-0-99-40.us-east-2.compute.internal Excluded Paused 4.14.0 -

= Update Health =
Message: Outdated nodes in a paused pool 'worker' will not be updated
Since: -
Level: Warning
Impact: Update Stalled
Reference: https://docs.openshift.com/container-platform/4.15/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues
Resources:
MachineConfigPools: worker
Description: Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator.
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,4 @@ ip-10-0-99-40.us-east-2.compute.internal Excluded Paused 4.14.0 -

= Update Health =
SINCE LEVEL IMPACT MESSAGE
- Warning Update Stalled Worker pool worker is paused | Outdated nodes in a paused pool will not be updated.
- Warning Update Stalled Outdated nodes in a paused pool 'worker' will not be updated
Loading