From a0200a12c026d2b4cd8342e8c3b3425c69ce73d7 Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Thu, 22 Feb 2024 15:40:52 +0100 Subject: [PATCH 01/10] `upgrade status`: --details flag that shows reference and description With --details, every insight will have two more fields - description and remediation URL (runbook, documentation...). Output is not a table, but a `oc describe`-like tree output. --- .../admin/upgrade/status/controlplane_test.go | 2 +- .../examples/4.14.1-degraded.detailed-output | 63 +++++++++++++ .../4.14.1-paused-worker-pool.detailed-output | 29 ++++++ ...ed-updating-multiple-pools.detailed-output | 40 ++++++++ ...-ec2-b02-cos-not-annotated.detailed-output | 29 ++++++ .../examples/4.15.0-ec2-early.detailed-output | 29 ++++++ ....0-ec2-unavailable-mco-20m.detailed-output | 29 ++++++ ...ted-pdb-prohibits-draining.detailed-output | 94 +++++++++++++++++++ .../examples/not-upgrading.detailed-output | 4 + pkg/cli/admin/upgrade/status/examples_test.go | 66 +++++++++---- pkg/cli/admin/upgrade/status/health.go | 80 +++++++++++++--- pkg/cli/admin/upgrade/status/health_test.go | 94 +++++++++++++------ pkg/cli/admin/upgrade/status/status.go | 5 +- 13 files changed, 499 insertions(+), 65 deletions(-) create mode 100644 pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output create mode 100644 pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output create mode 100644 pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output create mode 100644 pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output create mode 100644 pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output create mode 100644 pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output create mode 100644 pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output create mode 100644 pkg/cli/admin/upgrade/status/examples/not-upgrading.detailed-output diff --git a/pkg/cli/admin/upgrade/status/controlplane_test.go b/pkg/cli/admin/upgrade/status/controlplane_test.go index 9e4eb40044..54ec5ab002 100644 --- a/pkg/cli/admin/upgrade/status/controlplane_test.go +++ b/pkg/cli/admin/upgrade/status/controlplane_test.go @@ -129,7 +129,7 @@ var cvFixture = configv1.ClusterVersion{ }, } -var allowUnexportedInsightStructs = cmp.AllowUnexported(updateInsight{}, updateInsightScope{}, scopeResource{}, updateInsightImpact{}) +var allowUnexportedInsightStructs = cmp.AllowUnexported(updateInsight{}, updateInsightScope{}, scopeResource{}, updateInsightImpact{}, updateInsightRemediation{}) func TestAssessControlPlaneStatus_Operators(t *testing.T) { testCases := []struct { diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output new file mode 100644 index 0000000000..8b2720f2ef --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output @@ -0,0 +1,63 @@ +An update is in progress for 1h58m50s: Unable to apply 4.14.1: wait has exceeded 40 minutes for these operators: etcd, kube-apiserver + +Failing=True: + + Reason: ClusterOperatorsDegraded + Message: Cluster operators etcd, kube-apiserver are degraded + + += Control Plane = +Assessment: Progressing +Completion: 97% +Duration: 1h58m50s +Operator Status: 33 Total, 32 Available, 1 Progressing, 4 Degraded + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Pending +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded + +Worker Pool Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? + += Update Health = +Message: Cluster Operator kube-apiserver is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) + Since: 58m18s + Level: Error + Impact: API Availability + Reference: + Description: + +Message: Cluster Operator kube-controller-manager is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) + Since: 58m18s + Level: Error + Impact: API Availability + Reference: + Description: + +Message: Cluster Operator kube-scheduler is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) + Since: 58m18s + Level: Error + Impact: API Availability + Reference: + Description: + +Message: Cluster Operator etcd is degraded | EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] // EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy // NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) + Since: 58m38s + Level: Error + Impact: API Availability + Reference: + Description: + +Message: Cluster Operator control-plane-machine-set is unavailable | UnavailableReplicas: Missing 1 available replica(s) + Since: 1h0m17s + Level: Error + Impact: API Availability + Reference: + Description: diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output new file mode 100644 index 0000000000..553aa4e843 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output @@ -0,0 +1,29 @@ +An update is in progress for 6s: Working towards 4.14.1: 139 of 859 done (16% complete), waiting on kube-scheduler + += Control Plane = +Assessment: Progressing +Completion: 12% +Duration: 6s +Operator Status: 33 Total, 33 Available, 0 Progressing, 0 Degraded + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Excluded +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 3 Excluded, 0 Degraded + +Worker Pool Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Excluded Paused 4.14.0 - +ip-10-0-4-159.us-east-2.compute.internal Excluded Paused 4.14.0 - +ip-10-0-99-40.us-east-2.compute.internal Excluded Paused 4.14.0 - + += Update Health = +Message: Worker pool worker is paused | Outdated nodes in a paused pool will not be updated. + Since: - + Level: Warning + Impact: Update Stalled + Reference: + Description: diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output new file mode 100644 index 0000000000..901c06a79c --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output @@ -0,0 +1,40 @@ +An update is in progress for 14m4s: Working towards 4.14.1: 734 of 859 done (85% complete), waiting on machine-config + += Control Plane = +Assessment: Progressing +Completion: 97% +Duration: 14m4s +Operator Status: 33 Total, 32 Available, 1 Progressing, 0 Degraded + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Progressing +Completion: 0% +Worker Status: 3 Total, 2 Available, 1 Progressing, 3 Outdated, 1 Draining, 0 Excluded, 0 Degraded + +Worker Pool Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-4-159.us-east-2.compute.internal Progressing Draining 4.14.0 +30m +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.0 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0 ? + += Worker Pool = +Worker Pool: infra +Assessment: Progressing +Completion: 0% +Worker Status: 2 Total, 1 Available, 1 Progressing, 2 Outdated, 1 Draining, 0 Excluded, 0 Degraded + +Worker Pool Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-4-159-infra.us-east-2.compute.internal Progressing Draining 4.14.0 +30m +ip-10-0-20-162-infra.us-east-2.compute.internal Outdated Pending 4.14.0 ? + += Update Health = +Message: Upgrade is proceeding well + Since: 14m4s + Level: Info + Impact: None + Reference: + Description: diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output new file mode 100644 index 0000000000..49a42ff97c --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output @@ -0,0 +1,29 @@ +An update is in progress for 52m56s: Working towards 4.15.0-ec.2: 357 of 1021 done (34% complete), waiting up to 40 minutes on cluster-api + += Control Plane = +Assessment: Progressing +Completion: 43% +Duration: 52m56s +Operator Status: 7 Total, 7 Available, 0 Progressing, 0 Degraded + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Pending +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded + +Worker Pool Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? +ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? + += Update Health = +Message: Upgrade is proceeding well + Since: 52m56s + Level: Info + Impact: None + Reference: + Description: diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output new file mode 100644 index 0000000000..1d558de3cc --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output @@ -0,0 +1,29 @@ +An update is in progress for 1m29s: Working towards 4.15.0-ec.2: 106 of 863 done (12% complete), waiting on etcd, kube-apiserver + += Control Plane = +Assessment: Progressing +Completion: 3% +Duration: 1m29s +Operator Status: 33 Total, 33 Available, 2 Progressing, 0 Degraded + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Pending +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded + +Worker Pool Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.1 ? +ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.1 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.1 ? + += Update Health = +Message: Upgrade is proceeding well + Since: 1m29s + Level: Info + Impact: None + Reference: + Description: diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output new file mode 100644 index 0000000000..461f61a4ec --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output @@ -0,0 +1,29 @@ +An update is in progress for 58m53s: Working towards 4.15.0-ec.2: 110 of 863 done (12% complete), waiting up to 40 minutes on etcd + += Control Plane = +Assessment: Progressing +Completion: 97% +Duration: 58m53s +Operator Status: 33 Total, 31 Available, 1 Progressing, 1 Degraded + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Pending +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded + +Worker Pool Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? + += Update Health = +Message: Cluster Operator machine-config is unavailable | MachineConfigControllerFailed: Cluster not available for [{operator 4.14.0-rc.3}]: ControllerConfig.machineconfiguration.openshift.io "machine-config-controller" is invalid: [status.controllerCertificates[0].notAfter: Required value, status.controllerCertificates[0].notBefore: Required value, status.controllerCertificates[1].notAfter: Required value, status.controllerCertificates[1].notBefore: Required value, status.controllerCertificates[2].notAfter: Required value, status.controllerCertificates[2].notBefore: Required value, status.controllerCertificates[3].notAfter: Required value, status.controllerCertificates[3].notBefore: Required value, status.controllerCertificates[4].notAfter: Required value, status.controllerCertificates[4].notBefore: Required value, status.controllerCertificates[5].notAfter: Required value, status.controllerCertificates[5].notBefore: Required value, status.controllerCertificates[6].notAfter: Required value, status.controllerCertificates[6].notBefore: Required value, status.controllerCertificates[7].notAfter: Required value, status.controllerCertificates[7].notBefore: Required value, status.controllerCertificates[8].notAfter: Required value, status.controllerCertificates[8].notBefore: Required value, status.controllerCertificates[9].notAfter: Required value, status.controllerCertificates[9].notBefore: Required value, : Invalid value: "null": some validation rules were not checked because the object was invalid; correct the existing errors to complete validation] + Since: 20m24s + Level: Error + Impact: API Availability + Reference: + Description: diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output new file mode 100644 index 0000000000..97256b23fe --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output @@ -0,0 +1,94 @@ +An update is in progress for 4h3m46s: Error while reconciling 4.16.0-ec.3: the cluster operator machine-config is degraded + +Failing=True: + + Reason: ClusterOperatorDegraded + Message: Cluster operator machine-config is degraded + + += Control Plane = +Assessment: Completed +Completion: 100% +Duration: 3h30m31s +Operator Status: 36 Total, 36 Available, 0 Progressing, 1 Degraded + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Degraded +Completion: 39% +Worker Status: 59 Total, 46 Available, 5 Progressing, 36 Outdated, 12 Draining, 0 Excluded, 7 Degraded + +Worker Pool Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-c-jq5rk Unavailable Updated 4.16.0-ec.3 - Node is unavailable +build0-gstfj-ci-tests-worker-c-2kz4m Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-tests-worker-c-55hpj Progressing Draining 4.16.0-ec.2 +30m +... +Omitted additional 49 Total, 22 Completed, 46 Available, 3 Progressing, 27 Outdated, 3 Draining, 0 Excluded, and 0 Degraded nodes. +Pass along --details to see all information. + += Update Health = +Message: Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information + Since: - + Level: Error + Impact: Update Stalled + Reference: + Description: + +Message: Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information + Since: - + Level: Error + Impact: Update Stalled + Reference: + Description: + +Message: Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information + Since: - + Level: Error + Impact: Update Stalled + Reference: + Description: + +Message: Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information + Since: - + Level: Error + Impact: Update Stalled + Reference: + Description: + +Message: Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information + Since: - + Level: Error + Impact: Update Stalled + Reference: + Description: + +Message: Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information + Since: - + Level: Error + Impact: Update Stalled + Reference: + Description: + +Message: Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded | failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information + Since: - + Level: Error + Impact: Update Stalled + Reference: + Description: + +Message: Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable | Node is unavailable + Since: - + Level: Warning + Impact: Update Speed + Reference: + Description: diff --git a/pkg/cli/admin/upgrade/status/examples/not-upgrading.detailed-output b/pkg/cli/admin/upgrade/status/examples/not-upgrading.detailed-output new file mode 100644 index 0000000000..add8b8cab0 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/not-upgrading.detailed-output @@ -0,0 +1,4 @@ +The cluster version is not updating (Progressing=False). + + Reason: + Message: Cluster version is 4.14.1 diff --git a/pkg/cli/admin/upgrade/status/examples_test.go b/pkg/cli/admin/upgrade/status/examples_test.go index 7782c0d8cb..bf1e7ef4db 100644 --- a/pkg/cli/admin/upgrade/status/examples_test.go +++ b/pkg/cli/admin/upgrade/status/examples_test.go @@ -3,6 +3,7 @@ package status import ( "bytes" "context" + "fmt" "os" "path/filepath" "strings" @@ -11,9 +12,9 @@ import ( "github.com/google/go-cmp/cmp" ) -func compareWithFixture(t *testing.T, actualOut []byte, cvPath string) { +func compareWithFixture(t *testing.T, actualOut []byte, cvPath string, outputSuffix string) { t.Helper() - expectedOutPath := strings.Replace(cvPath, "-cv.yaml", ".output", 1) + expectedOutPath := strings.Replace(cvPath, "-cv.yaml", outputSuffix, 1) if update := os.Getenv("UPDATE"); update != "" { if err := os.WriteFile(expectedOutPath, actualOut, 0644); err != nil { @@ -24,10 +25,15 @@ func compareWithFixture(t *testing.T, actualOut []byte, cvPath string) { expectedOut, err := os.ReadFile(expectedOutPath) if err != nil { - t.Fatalf("Error when reading output fixture: %v", err) + if !os.IsNotExist(err) { + t.Fatalf("Error when reading output fixture: %v", err) + } else { + t.Fatalf("Output file %s does not exist. You may rerun this test with UPDATE=true to create output file with the following actual output:\n%s", expectedOutPath, actualOut) + } } + if diff := cmp.Diff(string(expectedOut), string(actualOut)); diff != "" { - t.Errorf("Output differs from expected:\n%s", diff) + t.Errorf("Output differs from expected (%s):\n%s", filepath.Base(expectedOutPath), diff) } } @@ -37,25 +43,47 @@ func TestExamples(t *testing.T) { t.Fatalf("Error when listing examples: %v", err) } + variants := []struct { + name string + detailed bool + outputSuffix string + }{ + { + name: "normal output", + detailed: false, + outputSuffix: ".output", + }, + { + name: "detailed output", + detailed: true, + outputSuffix: ".detailed-output", + }, + } + for _, cv := range cvs { cv := cv - t.Run(cv, func(t *testing.T) { - t.Parallel() - - opts := &options{mockData: mockData{cvPath: cv}} - if err := opts.Complete(nil, nil, nil); err != nil { - t.Fatalf("Error when completing options: %v", err) - } + for _, variant := range variants { + variant := variant + t.Run(fmt.Sprintf("%s-%s", cv, variant.name), func(t *testing.T) { + t.Parallel() + opts := &options{ + mockData: mockData{cvPath: cv}, + detailedOutput: variant.detailed, + } + if err := opts.Complete(nil, nil, nil); err != nil { + t.Fatalf("Error when completing options: %v", err) + } - var stdout, stderr bytes.Buffer - opts.Out = &stdout - opts.ErrOut = &stderr + var stdout, stderr bytes.Buffer + opts.Out = &stdout + opts.ErrOut = &stderr - if err := opts.Run(context.Background()); err != nil { - t.Fatalf("Error when running: %v", err) - } + if err := opts.Run(context.Background()); err != nil { + t.Fatalf("Error when running: %v", err) + } - compareWithFixture(t, stdout.Bytes(), cv) - }) + compareWithFixture(t, stdout.Bytes(), cv, variant.outputSuffix) + }) + } } } diff --git a/pkg/cli/admin/upgrade/status/health.go b/pkg/cli/admin/upgrade/status/health.go index 0bd0484662..78124c0f46 100644 --- a/pkg/cli/admin/upgrade/status/health.go +++ b/pkg/cli/admin/upgrade/status/health.go @@ -1,6 +1,7 @@ package status import ( + "fmt" "io" "sort" "strings" @@ -88,15 +89,21 @@ const ( ) type updateInsightImpact struct { - level impactLevel - impactType impactType - summary string + level impactLevel + impactType impactType + summary string + description string +} + +type updateInsightRemediation struct { + reference string } type updateInsight struct { - startedAt time.Time - scope updateInsightScope - impact updateInsightImpact + startedAt time.Time + scope updateInsightScope + impact updateInsightImpact + remediation updateInsightRemediation } type updateHealthData struct { @@ -155,17 +162,62 @@ func stringSince(health *updateHealthData, insight updateInsight) string { return shortDuration(health.evaluatedAt.Sub(insight.startedAt).Truncate(time.Second)) } -func (i *updateHealthData) Write(w io.Writer) error { +type displayItem struct { + since string + level string + impact string + message string + description string + reference string +} + +func (i *updateHealthData) Write(w io.Writer, detailed bool) error { _, _ = w.Write([]byte("= Update Health =\n")) - tabw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) - _, _ = tabw.Write([]byte("SINCE\tLEVEL\tIMPACT\tMESSAGE\n")) + + displayData := make([]displayItem, 0, len(i.insights)) + for _, insight := range i.insights { - _, _ = tabw.Write([]byte(stringSince(i, insight) + "\t")) - _, _ = tabw.Write([]byte(insight.impact.level.String() + "\t")) - _, _ = tabw.Write([]byte(insight.impact.impactType + "\t")) - _, _ = tabw.Write([]byte(insight.impact.summary + "\n")) + displayData = append(displayData, displayItem{ + since: stringSince(i, insight), + level: insight.impact.level.String(), + impact: string(insight.impact.impactType), + message: insight.impact.summary, + description: insight.impact.description, + reference: insight.remediation.reference, + }) + } + if detailed { + detailedOutput(w, displayData) + } else { + tabulatedOutput(w, displayData) } - tabw.Flush() return nil } + +func detailedOutput(w io.Writer, items []displayItem) { + pad := len("Description: ") + for i, item := range items { + _, _ = w.Write([]byte(fmt.Sprintf("Message: %s\n", item.message))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Since:", item.since))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Level:", item.level))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Impact:", item.impact))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Reference:", item.reference))) + _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Description:", item.description))) + if len(items) > i+1 { + _, _ = w.Write([]byte("\n")) + } + } +} + +func tabulatedOutput(w io.Writer, items []displayItem) { + tabw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + _, _ = tabw.Write([]byte("SINCE\tLEVEL\tIMPACT\tMESSAGE\n")) + for _, item := range items { + _, _ = tabw.Write([]byte(item.since + "\t")) + _, _ = tabw.Write([]byte(item.level + "\t")) + _, _ = tabw.Write([]byte(item.impact + "\t")) + _, _ = tabw.Write([]byte(item.message + "\n")) + } + tabw.Flush() +} diff --git a/pkg/cli/admin/upgrade/status/health_test.go b/pkg/cli/admin/upgrade/status/health_test.go index 36a04d389a..07f6b4ba28 100644 --- a/pkg/cli/admin/upgrade/status/health_test.go +++ b/pkg/cli/admin/upgrade/status/health_test.go @@ -14,42 +14,52 @@ var insights = []updateInsight{ { startedAt: now, impact: updateInsightImpact{ - level: infoImpactLevel, - impactType: noneImpactType, - summary: "Something with no impact that happened right now", + level: infoImpactLevel, + impactType: noneImpactType, + summary: "Something with no impact that happened right now", + description: "This is a test", }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com"}, }, { startedAt: now.Add(-10 * time.Second), impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Something that slows the update", + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Something that slows the update", + description: "Your pathetic hardware is slowing OCP down", }, + remediation: updateInsightRemediation{reference: "https://cs.wikipedia.org/wiki/Hardware"}, }, { startedAt: now.Add(-20 * time.Second), impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: clusterCapacityImpactType, - summary: "Something that limits cluster capacity", + level: errorImpactLevel, + impactType: clusterCapacityImpactType, + summary: "Something that limits cluster capacity", + description: "Autoscaler is disabled, you should enable it", }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com/container-platform/4.14/nodes/pods/nodes-pods-autoscaling.html"}, }, { startedAt: now.Add(-5 * time.Second), impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Something that broke API and happened recently", + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Something that broke API and happened recently", + description: "Only one auth replica is available", }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com/container-platform/4.14/authentication/understanding-authentication.html"}, }, { startedAt: now.Add(-time.Hour), impact: updateInsightImpact{ - level: criticalInfoLevel, - impactType: dataLossImpactType, - summary: "KTHXBAI DATA", + level: criticalInfoLevel, + impactType: dataLossImpactType, + summary: "KTHXBAI DATA", + description: "You lost all your data, hope you have a backup", }, + remediation: updateInsightRemediation{reference: "https://en.wikipedia.org/wiki/Backup/"}, }, } @@ -89,26 +99,50 @@ func TestAssessUpdateInsights_NoInsightsCreatesAllIsWellInfo(t *testing.T) { }, } - if diff := cmp.Diff(expected, healthData, cmp.AllowUnexported(updateHealthData{}, updateInsight{}, updateInsightScope{}, updateInsightImpact{})); diff != "" { + if diff := cmp.Diff(expected, healthData, cmp.AllowUnexported(updateHealthData{}, updateInsight{}, updateInsightScope{}, updateInsightImpact{}, updateInsightRemediation{})); diff != "" { t.Fatalf("Output differs from expected :\n%s", diff) } } func TestUpdateHealthData_Write(t *testing.T) { - var w strings.Builder - healthData := assessUpdateInsights(insights, 2*time.Hour, now) - if err := healthData.Write(&w); err != nil { - t.Fatalf("Unexpected error: %v", err) + testCases := []struct { + name string + detailed bool + expected string + }{ + { + name: "not detailed", + detailed: false, + expected: "= Update Health =\n" + + "SINCE LEVEL IMPACT MESSAGE\n" + + "1h Critical Data Loss KTHXBAI DATA\n" + + "5s Error API Availability Something that broke API and happened recently\n" + + "20s Error Cluster Capacity Something that limits cluster capacity\n" + + "10s Warning Update Speed Something that slows the update\n" + + "0s Info None Something with no impact that happened right now\n", + }, + { + name: "detailed", + detailed: true, + expected: "= Update Health =\n" + + "Message: KTHXBAI DATA\n Since: 1h\n Level: Critical\n Impact: Data Loss\n Reference: https://en.wikipedia.org/wiki/Backup/\n Description: You lost all your data, hope you have a backup\n\n" + + "Message: Something that broke API and happened recently\n Since: 5s\n Level: Error\n Impact: API Availability\n Reference: https://docs.openshift.com/container-platform/4.14/authentication/understanding-authentication.html\n Description: Only one auth replica is available\n\n" + + "Message: Something that limits cluster capacity\n Since: 20s\n Level: Error\n Impact: Cluster Capacity\n Reference: https://docs.openshift.com/container-platform/4.14/nodes/pods/nodes-pods-autoscaling.html\n Description: Autoscaler is disabled, you should enable it\n\n" + + "Message: Something that slows the update\n Since: 10s\n Level: Warning\n Impact: Update Speed\n Reference: https://cs.wikipedia.org/wiki/Hardware\n Description: Your pathetic hardware is slowing OCP down\n\n" + + "Message: Something with no impact that happened right now\n Since: 0s\n Level: Info\n Impact: None\n Reference: https://docs.openshift.com\n Description: This is a test\n", + }, } - var expected = "= Update Health =\n" + - "SINCE LEVEL IMPACT MESSAGE\n" + - "1h Critical Data Loss KTHXBAI DATA\n" + - "5s Error API Availability Something that broke API and happened recently\n" + - "20s Error Cluster Capacity Something that limits cluster capacity\n" + - "10s Warning Update Speed Something that slows the update\n" + - "0s Info None Something with no impact that happened right now\n" - - if diff := cmp.Diff(expected, w.String()); diff != "" { - t.Fatalf("Output differs from expected :\n%s", diff) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var w strings.Builder + healthData := assessUpdateInsights(insights, 2*time.Hour, now) + if err := healthData.Write(&w, tc.detailed); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if diff := cmp.Diff(tc.expected, w.String()); diff != "" { + t.Fatalf("Output differs from expected :\n%s", diff) + } + }) } + } diff --git a/pkg/cli/admin/upgrade/status/status.go b/pkg/cli/admin/upgrade/status/status.go index c50e533f79..c979a42fad 100644 --- a/pkg/cli/admin/upgrade/status/status.go +++ b/pkg/cli/admin/upgrade/status/status.go @@ -49,6 +49,7 @@ func New(f kcmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Command // TODO: We can remove these flags once the idea about `oc adm upgrade status` stabilizes and the command // is promoted out of the OC_ENABLE_CMD_UPGRADE_STATUS feature gate flags.StringVar(&o.mockData.cvPath, "mock-clusterversion", "", "Path to a YAML ClusterVersion object to use for testing (will be removed later). Files in the same directory with the same name and suffixes -co.yaml, -mcp.yaml, -mc.yaml, and -node.yaml are required.") + flags.BoolVar(&o.detailedOutput, "details", false, "Show upgrade health insights with detailed information and remediating links") return cmd } @@ -61,6 +62,8 @@ type options struct { ConfigClient configv1client.Interface CoreClient corev1client.CoreV1Interface MachineConfigClient machineconfigv1client.Interface + + detailedOutput bool } func (o *options) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { @@ -255,7 +258,7 @@ func (o *options) Run(ctx context.Context) error { fmt.Fprintf(o.Out, "\n") upgradeHealth := assessUpdateInsights(updateInsights, updatingFor, now) - _ = upgradeHealth.Write(o.Out) + _ = upgradeHealth.Write(o.Out, o.detailedOutput) return nil } From 99568374519cded36b776f71d668231352036a8c Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Thu, 22 Feb 2024 15:59:01 +0100 Subject: [PATCH 02/10] `upgrade status`: populate ref & description for CO insights Move the condition message from the insight summary to its description (which is only shown with --details). Remediation links point to o/runbooks runbook for the matching alert. --- pkg/cli/admin/upgrade/status/controlplane.go | 16 +++-- .../admin/upgrade/status/controlplane_test.go | 40 +++++++---- .../examples/4.14.1-degraded.detailed-output | 30 ++++---- .../status/examples/4.14.1-degraded.output | 10 +-- .../4.14.1-paused-worker-pool.detailed-output | 6 +- .../examples/4.14.1-paused-worker-pool.output | 2 +- ....0-ec2-unavailable-mco-20m.detailed-output | 6 +- .../4.15.0-ec2-unavailable-mco-20m.output | 2 +- ...ted-pdb-prohibits-draining.detailed-output | 48 ++++++------- ...lane-updated-pdb-prohibits-draining.output | 16 ++--- pkg/cli/admin/upgrade/status/workerpool.go | 30 +++++--- .../admin/upgrade/status/workerpool_test.go | 70 +++++++++++++------ 12 files changed, 165 insertions(+), 111 deletions(-) diff --git a/pkg/cli/admin/upgrade/status/controlplane.go b/pkg/cli/admin/upgrade/status/controlplane.go index 3ef4e7b994..c3c071bb76 100644 --- a/pkg/cli/admin/upgrade/status/controlplane.go +++ b/pkg/cli/admin/upgrade/status/controlplane.go @@ -48,10 +48,12 @@ func coInsights(name string, available v1.ClusterOperatorStatusCondition, degrad startedAt: available.LastTransitionTime.Time, scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: name}}}, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: apiAvailabilityImpactType, - summary: fmt.Sprintf("Cluster Operator %s is unavailable | %s: %s", name, available.Reason, strings.ReplaceAll(available.Message, "\n", ` // `)), + level: warningImpactLevel, + impactType: apiAvailabilityImpactType, + summary: fmt.Sprintf("Cluster Operator %s is unavailable (%s)", name, available.Reason), + description: strings.ReplaceAll(available.Message, "\n", ` // `), }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"}, } if evaluated.After(available.LastTransitionTime.Time.Add(unavailableErrorThreshold)) { insight.impact.level = errorImpactLevel @@ -63,10 +65,12 @@ func coInsights(name string, available v1.ClusterOperatorStatusCondition, degrad startedAt: degraded.LastTransitionTime.Time, scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: name}}}, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: apiAvailabilityImpactType, - summary: fmt.Sprintf("Cluster Operator %s is degraded | %s: %s", name, degraded.Reason, strings.ReplaceAll(degraded.Message, "\n", ` // `)), + level: warningImpactLevel, + impactType: apiAvailabilityImpactType, + summary: fmt.Sprintf("Cluster Operator %s is degraded (%s)", name, degraded.Reason), + description: strings.ReplaceAll(degraded.Message, "\n", ` // `), }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md"}, } if evaluated.After(degraded.LastTransitionTime.Time.Add(degradedErrorThreshold)) { insight.impact.level = errorImpactLevel diff --git a/pkg/cli/admin/upgrade/status/controlplane_test.go b/pkg/cli/admin/upgrade/status/controlplane_test.go index 54ec5ab002..132f755781 100644 --- a/pkg/cli/admin/upgrade/status/controlplane_test.go +++ b/pkg/cli/admin/upgrade/status/controlplane_test.go @@ -404,19 +404,23 @@ func TestCoInsights(t *testing.T) { startedAt: anchorTime.Add(-unavailableWarningThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Cluster Operator testOperator is unavailable | Broken: Operator is broken", + level: warningImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Cluster Operator testOperator is unavailable (Broken)", + description: "Operator is broken", }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"}, }, { startedAt: anchorTime.Add(-degradedWarningThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Cluster Operator testOperator is degraded | Slow: Networking is hard", + level: warningImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Cluster Operator testOperator is degraded (Slow)", + description: "Networking is hard", }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md"}, }, }, }, @@ -441,19 +445,23 @@ func TestCoInsights(t *testing.T) { startedAt: anchorTime.Add(-unavailableErrorThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Cluster Operator testOperator is unavailable | Broken: Operator is broken", + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Cluster Operator testOperator is unavailable (Broken)", + description: "Operator is broken", }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"}, }, { startedAt: anchorTime.Add(-degradedErrorThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: apiAvailabilityImpactType, - summary: "Cluster Operator testOperator is degraded | Slow: Networking is hard", + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Cluster Operator testOperator is degraded (Slow)", + description: "Networking is hard", }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md"}, }, }, }, @@ -475,10 +483,12 @@ func TestCoInsights(t *testing.T) { startedAt: anchorTime.Add(-unavailableErrorThreshold).Add(-time.Second), scope: updateInsightScope{scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindClusterOperator, name: "testOperator"}}}, impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: apiAvailabilityImpactType, - summary: `Cluster Operator testOperator is unavailable | Broken: Operator is broken // and message has linebreaks`, + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: `Cluster Operator testOperator is unavailable (Broken)`, + description: "Operator is broken // and message has linebreaks", }, + remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"}, }, }, }, diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output index 8b2720f2ef..196218a594 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output @@ -27,37 +27,37 @@ ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? = Update Health = -Message: Cluster Operator kube-apiserver is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) +Message: Cluster Operator kube-apiserver is degraded (NodeController_MasterNodesReady) Since: 58m18s Level: Error Impact: API Availability - Reference: - Description: + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -Message: Cluster Operator kube-controller-manager is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) +Message: Cluster Operator kube-controller-manager is degraded (NodeController_MasterNodesReady) Since: 58m18s Level: Error Impact: API Availability - Reference: - Description: + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -Message: Cluster Operator kube-scheduler is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) +Message: Cluster Operator kube-scheduler is degraded (NodeController_MasterNodesReady) Since: 58m18s Level: Error Impact: API Availability - Reference: - Description: + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -Message: Cluster Operator etcd is degraded | EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] // EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy // NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) +Message: Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady) Since: 58m38s Level: Error Impact: API Availability - Reference: - Description: + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Description: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] // EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy // NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -Message: Cluster Operator control-plane-machine-set is unavailable | UnavailableReplicas: Missing 1 available replica(s) +Message: Cluster Operator control-plane-machine-set is unavailable (UnavailableReplicas) Since: 1h0m17s Level: Error Impact: API Availability - Reference: - Description: + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md + Description: Missing 1 available replica(s) diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output index 35aa29b7c7..e832e2ca1a 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output @@ -34,8 +34,8 @@ ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 = Update Health = SINCE LEVEL IMPACT MESSAGE -58m18s Error API Availability Cluster Operator kube-apiserver is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -58m18s Error API Availability Cluster Operator kube-controller-manager is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -58m18s Error API Availability Cluster Operator kube-scheduler is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -58m38s Error API Availability Cluster Operator etcd is degraded | EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] // EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy // NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) -1h0m17s Error API Availability Cluster Operator control-plane-machine-set is unavailable | UnavailableReplicas: Missing 1 available replica(s) +58m18s Error API Availability Cluster Operator kube-apiserver is degraded (NodeController_MasterNodesReady) +58m18s Error API Availability Cluster Operator kube-controller-manager is degraded (NodeController_MasterNodesReady) +58m18s Error API Availability Cluster Operator kube-scheduler is degraded (NodeController_MasterNodesReady) +58m38s Error API Availability Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady) +1h0m17s Error API Availability Cluster Operator control-plane-machine-set is unavailable (UnavailableReplicas) diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output index 553aa4e843..c238a7a4c6 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output @@ -21,9 +21,9 @@ ip-10-0-4-159.us-east-2.compute.internal Excluded Paused 4.14.0 - ip-10-0-99-40.us-east-2.compute.internal Excluded Paused 4.14.0 - = Update Health = -Message: Worker pool worker is paused | Outdated nodes in a paused pool will not be updated. +Message: Worker pool worker is paused Since: - Level: Warning Impact: Update Stalled - Reference: - Description: + Reference: https://docs.openshift.com/container-platform/4.15/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues + Description: Outdated nodes in a paused pool will not be updated diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output index dd22bdd9e8..0689526a9f 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output @@ -28,4 +28,4 @@ ip-10-0-99-40.us-east-2.compute.internal Excluded Paused 4.14.0 - = Update Health = SINCE LEVEL IMPACT MESSAGE -- Warning Update Stalled Worker pool worker is paused | Outdated nodes in a paused pool will not be updated. +- Warning Update Stalled Worker pool worker is paused diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output index 461f61a4ec..a5180b0cfd 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output @@ -21,9 +21,9 @@ ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? = Update Health = -Message: Cluster Operator machine-config is unavailable | MachineConfigControllerFailed: Cluster not available for [{operator 4.14.0-rc.3}]: ControllerConfig.machineconfiguration.openshift.io "machine-config-controller" is invalid: [status.controllerCertificates[0].notAfter: Required value, status.controllerCertificates[0].notBefore: Required value, status.controllerCertificates[1].notAfter: Required value, status.controllerCertificates[1].notBefore: Required value, status.controllerCertificates[2].notAfter: Required value, status.controllerCertificates[2].notBefore: Required value, status.controllerCertificates[3].notAfter: Required value, status.controllerCertificates[3].notBefore: Required value, status.controllerCertificates[4].notAfter: Required value, status.controllerCertificates[4].notBefore: Required value, status.controllerCertificates[5].notAfter: Required value, status.controllerCertificates[5].notBefore: Required value, status.controllerCertificates[6].notAfter: Required value, status.controllerCertificates[6].notBefore: Required value, status.controllerCertificates[7].notAfter: Required value, status.controllerCertificates[7].notBefore: Required value, status.controllerCertificates[8].notAfter: Required value, status.controllerCertificates[8].notBefore: Required value, status.controllerCertificates[9].notAfter: Required value, status.controllerCertificates[9].notBefore: Required value, : Invalid value: "null": some validation rules were not checked because the object was invalid; correct the existing errors to complete validation] +Message: Cluster Operator machine-config is unavailable (MachineConfigControllerFailed) Since: 20m24s Level: Error Impact: API Availability - Reference: - Description: + Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md + Description: Cluster not available for [{operator 4.14.0-rc.3}]: ControllerConfig.machineconfiguration.openshift.io "machine-config-controller" is invalid: [status.controllerCertificates[0].notAfter: Required value, status.controllerCertificates[0].notBefore: Required value, status.controllerCertificates[1].notAfter: Required value, status.controllerCertificates[1].notBefore: Required value, status.controllerCertificates[2].notAfter: Required value, status.controllerCertificates[2].notBefore: Required value, status.controllerCertificates[3].notAfter: Required value, status.controllerCertificates[3].notBefore: Required value, status.controllerCertificates[4].notAfter: Required value, status.controllerCertificates[4].notBefore: Required value, status.controllerCertificates[5].notAfter: Required value, status.controllerCertificates[5].notBefore: Required value, status.controllerCertificates[6].notAfter: Required value, status.controllerCertificates[6].notBefore: Required value, status.controllerCertificates[7].notAfter: Required value, status.controllerCertificates[7].notBefore: Required value, status.controllerCertificates[8].notAfter: Required value, status.controllerCertificates[8].notBefore: Required value, status.controllerCertificates[9].notAfter: Required value, status.controllerCertificates[9].notBefore: Required value, : Invalid value: "null": some validation rules were not checked because the object was invalid; correct the existing errors to complete validation] diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output index af5e71c194..1f241b314c 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output @@ -28,4 +28,4 @@ ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 = Update Health = SINCE LEVEL IMPACT MESSAGE -20m24s Error API Availability Cluster Operator machine-config is unavailable | MachineConfigControllerFailed: Cluster not available for [{operator 4.14.0-rc.3}]: ControllerConfig.machineconfiguration.openshift.io "machine-config-controller" is invalid: [status.controllerCertificates[0].notAfter: Required value, status.controllerCertificates[0].notBefore: Required value, status.controllerCertificates[1].notAfter: Required value, status.controllerCertificates[1].notBefore: Required value, status.controllerCertificates[2].notAfter: Required value, status.controllerCertificates[2].notBefore: Required value, status.controllerCertificates[3].notAfter: Required value, status.controllerCertificates[3].notBefore: Required value, status.controllerCertificates[4].notAfter: Required value, status.controllerCertificates[4].notBefore: Required value, status.controllerCertificates[5].notAfter: Required value, status.controllerCertificates[5].notBefore: Required value, status.controllerCertificates[6].notAfter: Required value, status.controllerCertificates[6].notBefore: Required value, status.controllerCertificates[7].notAfter: Required value, status.controllerCertificates[7].notBefore: Required value, status.controllerCertificates[8].notAfter: Required value, status.controllerCertificates[8].notBefore: Required value, status.controllerCertificates[9].notAfter: Required value, status.controllerCertificates[9].notBefore: Required value, : Invalid value: "null": some validation rules were not checked because the object was invalid; correct the existing errors to complete validation] +20m24s Error API Availability Cluster Operator machine-config is unavailable (MachineConfigControllerFailed) diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output index 97256b23fe..95a9593161 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output @@ -37,58 +37,58 @@ Omitted additional 49 Total, 22 Completed, 46 Available, 3 Progressing, 27 Outda Pass along --details to see all information. = Update Health = -Message: Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information +Message: Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded Since: - Level: Error Impact: Update Stalled - Reference: - Description: + Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information -Message: Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information +Message: Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded Since: - Level: Error Impact: Update Stalled - Reference: - Description: + Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information -Message: Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information +Message: Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded Since: - Level: Error Impact: Update Stalled - Reference: - Description: + Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information -Message: Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information +Message: Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded Since: - Level: Error Impact: Update Stalled - Reference: - Description: + Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information -Message: Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information +Message: Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded Since: - Level: Error Impact: Update Stalled - Reference: - Description: + Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information -Message: Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information +Message: Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded Since: - Level: Error Impact: Update Stalled - Reference: - Description: + Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information -Message: Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded | failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information +Message: Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded Since: - Level: Error Impact: Update Stalled - Reference: - Description: + Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information -Message: Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable | Node is unavailable +Message: Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable Since: - Level: Warning Impact: Update Speed - Reference: - Description: + Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Description: Node is unavailable diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output index ebcd394dd6..18da08305d 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output @@ -44,11 +44,11 @@ Pass along --details to see all information. = Update Health = SINCE LEVEL IMPACT MESSAGE -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information -- Error Update Stalled Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded | failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information -- Warning Update Speed Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable | Node is unavailable +- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded +- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded +- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded +- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded +- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded +- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded +- Error Update Stalled Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded +- Warning Update Speed Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable diff --git a/pkg/cli/admin/upgrade/status/workerpool.go b/pkg/cli/admin/upgrade/status/workerpool.go index d61d26efad..281de1a63e 100644 --- a/pkg/cli/admin/upgrade/status/workerpool.go +++ b/pkg/cli/admin/upgrade/status/workerpool.go @@ -332,9 +332,13 @@ func nodeInsights(pool mcfgv1.MachineConfigPool, node corev1.Node, reason string resources: []scopeResource{{kind: scopeKindNode, name: node.Name}}, }, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: fmt.Sprintf("Node %s is unavailable | %s", node.Name, reason), + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: fmt.Sprintf("Node %s is unavailable", node.Name), + description: reason, + }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", }, }) } @@ -346,9 +350,13 @@ func nodeInsights(pool mcfgv1.MachineConfigPool, node corev1.Node, reason string resources: []scopeResource{{kind: scopeKindNode, name: node.Name}}, }, impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: updateStalledImpactType, - summary: fmt.Sprintf("Node %s is degraded | %s", node.Name, reason), + level: errorImpactLevel, + impactType: updateStalledImpactType, + summary: fmt.Sprintf("Node %s is degraded", node.Name), + description: reason, + }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", }, }) } @@ -431,9 +439,13 @@ func machineConfigPoolInsights(poolDisplay poolDisplayData, pool mcfgv1.MachineC resources: []scopeResource{{kind: scopeKindMachineConfigPool, name: pool.Name}}, }, impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateStalledImpactType, - summary: fmt.Sprintf("Worker pool %s is paused | Outdated nodes in a paused pool will not be updated.", pool.Name), + level: warningImpactLevel, + impactType: updateStalledImpactType, + summary: fmt.Sprintf("Worker pool %s is paused", pool.Name), + description: "Outdated nodes in a paused pool will not be updated", + }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues", }, }) } diff --git a/pkg/cli/admin/upgrade/status/workerpool_test.go b/pkg/cli/admin/upgrade/status/workerpool_test.go index fcf5990eb5..6ae70ae501 100644 --- a/pkg/cli/admin/upgrade/status/workerpool_test.go +++ b/pkg/cli/admin/upgrade/status/workerpool_test.go @@ -464,14 +464,18 @@ func Test_assessNodesStatus(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: updateStalledImpactType, - summary: "Node a is degraded | PDB prohibits draining", + level: errorImpactLevel, + impactType: updateStalledImpactType, + summary: "Node a is degraded", + description: "PDB prohibits draining", }, scope: updateInsightScope{ scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }, }, @@ -502,14 +506,18 @@ func Test_assessNodesStatus(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Node a is unavailable | Node is unavailable", + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Node a is unavailable", + description: "Node is unavailable", }, scope: updateInsightScope{ scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }}, } @@ -677,14 +685,18 @@ func Test_nodeInsights(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Node a is unavailable | Node is unavailable", + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Node a is unavailable", + description: "Node is unavailable", }, scope: updateInsightScope{ scopeType: scopeTypeControlPlane, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }, }, @@ -699,14 +711,18 @@ func Test_nodeInsights(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Node a is unavailable | Node is unavailable", + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Node a is unavailable", + description: "Node is unavailable", }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }, }, @@ -723,14 +739,18 @@ func Test_nodeInsights(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: updateStalledImpactType, - summary: "Node a is degraded | PDB prohibits draining", + level: errorImpactLevel, + impactType: updateStalledImpactType, + summary: "Node a is degraded", + description: "PDB prohibits draining", }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }, }, @@ -957,14 +977,18 @@ func Test_assessMachineConfigPool(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateStalledImpactType, - summary: "Worker pool worker is paused | Outdated nodes in a paused pool will not be updated.", + level: warningImpactLevel, + impactType: updateStalledImpactType, + summary: "Worker pool worker is paused", + description: "Outdated nodes in a paused pool will not be updated", }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, resources: []scopeResource{{kind: scopeKindMachineConfigPool, name: "worker"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues", + }, }, }}, } @@ -1049,14 +1073,18 @@ func Test_machineConfigPoolInsights(t *testing.T) { expectedUpdateInsight: []updateInsight{ { impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateStalledImpactType, - summary: "Worker pool worker is paused | Outdated nodes in a paused pool will not be updated.", + level: warningImpactLevel, + impactType: updateStalledImpactType, + summary: "Worker pool worker is paused", + description: "Outdated nodes in a paused pool will not be updated", }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, resources: []scopeResource{{kind: scopeKindMachineConfigPool, name: "worker"}}, }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues", + }, }, }, }, From 02f056606e77f7867708d673851e8bf60f0b9be2 Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Thu, 22 Feb 2024 16:08:19 +0100 Subject: [PATCH 03/10] `upgrade status`: keep linebreaks in description but respect indent Condition messages are propagated via an insight description, which is only shown with `--details` in a tree-like output. This allows us to keep linebreaks originally present in the message, we just need to pad each line enough to align it with the indentation of the message start. --- pkg/cli/admin/upgrade/status/controlplane.go | 4 ++-- pkg/cli/admin/upgrade/status/controlplane_test.go | 2 +- .../upgrade/status/examples/4.14.1-degraded.detailed-output | 4 +++- pkg/cli/admin/upgrade/status/health.go | 2 ++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/cli/admin/upgrade/status/controlplane.go b/pkg/cli/admin/upgrade/status/controlplane.go index c3c071bb76..98c2596a39 100644 --- a/pkg/cli/admin/upgrade/status/controlplane.go +++ b/pkg/cli/admin/upgrade/status/controlplane.go @@ -51,7 +51,7 @@ func coInsights(name string, available v1.ClusterOperatorStatusCondition, degrad level: warningImpactLevel, impactType: apiAvailabilityImpactType, summary: fmt.Sprintf("Cluster Operator %s is unavailable (%s)", name, available.Reason), - description: strings.ReplaceAll(available.Message, "\n", ` // `), + description: available.Message, }, remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"}, } @@ -68,7 +68,7 @@ func coInsights(name string, available v1.ClusterOperatorStatusCondition, degrad level: warningImpactLevel, impactType: apiAvailabilityImpactType, summary: fmt.Sprintf("Cluster Operator %s is degraded (%s)", name, degraded.Reason), - description: strings.ReplaceAll(degraded.Message, "\n", ` // `), + description: degraded.Message, }, remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md"}, } diff --git a/pkg/cli/admin/upgrade/status/controlplane_test.go b/pkg/cli/admin/upgrade/status/controlplane_test.go index 132f755781..c80f3035de 100644 --- a/pkg/cli/admin/upgrade/status/controlplane_test.go +++ b/pkg/cli/admin/upgrade/status/controlplane_test.go @@ -486,7 +486,7 @@ func TestCoInsights(t *testing.T) { level: errorImpactLevel, impactType: apiAvailabilityImpactType, summary: `Cluster Operator testOperator is unavailable (Broken)`, - description: "Operator is broken // and message has linebreaks", + description: "Operator is broken\nand message has linebreaks", }, remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"}, }, diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output index 196218a594..3cd50a1079 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output @@ -53,7 +53,9 @@ Message: Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpo Level: Error Impact: API Availability Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md - Description: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] // EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy // NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) + Description: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] + , EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy + , NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) Message: Cluster Operator control-plane-machine-set is unavailable (UnavailableReplicas) Since: 1h0m17s diff --git a/pkg/cli/admin/upgrade/status/health.go b/pkg/cli/admin/upgrade/status/health.go index 78124c0f46..84e7efd788 100644 --- a/pkg/cli/admin/upgrade/status/health.go +++ b/pkg/cli/admin/upgrade/status/health.go @@ -203,6 +203,8 @@ func detailedOutput(w io.Writer, items []displayItem) { _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Level:", item.level))) _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Impact:", item.impact))) _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Reference:", item.reference))) + // Respect the " Description: " indentation when description has linebreaks + item.description = strings.ReplaceAll(item.description, "\n", fmt.Sprintf("\n%s, ", strings.Repeat(" ", pad+2))) _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Description:", item.description))) if len(items) > i+1 { _, _ = w.Write([]byte("\n")) From 9f16602f8f7e20f57b357781c06c106921a7dcce Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Thu, 22 Feb 2024 16:35:11 +0100 Subject: [PATCH 04/10] `upgrade status`: ignore incomplete insights Incomplete insights are not not actionable enough to show them to the user, so we filter them out while processing. --- pkg/cli/admin/upgrade/status/controlplane.go | 8 +++ pkg/cli/admin/upgrade/status/health.go | 20 ++++++- pkg/cli/admin/upgrade/status/health_test.go | 59 ++++++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/pkg/cli/admin/upgrade/status/controlplane.go b/pkg/cli/admin/upgrade/status/controlplane.go index 98c2596a39..a78ec102b5 100644 --- a/pkg/cli/admin/upgrade/status/controlplane.go +++ b/pkg/cli/admin/upgrade/status/controlplane.go @@ -55,6 +55,10 @@ func coInsights(name string, available v1.ClusterOperatorStatusCondition, degrad }, remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md"}, } + if available.Message == "" { + // Backfill the description if CO doesn't provide one + insight.impact.description = "" + } if evaluated.After(available.LastTransitionTime.Time.Add(unavailableErrorThreshold)) { insight.impact.level = errorImpactLevel } @@ -72,6 +76,10 @@ func coInsights(name string, available v1.ClusterOperatorStatusCondition, degrad }, remediation: updateInsightRemediation{reference: "https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md"}, } + if degraded.Message == "" { + // Backfill the description if CO doesn't provide one + insight.impact.description = "" + } if evaluated.After(degraded.LastTransitionTime.Time.Add(degradedErrorThreshold)) { insight.impact.level = errorImpactLevel } diff --git a/pkg/cli/admin/upgrade/status/health.go b/pkg/cli/admin/upgrade/status/health.go index 84e7efd788..8f4aa543ba 100644 --- a/pkg/cli/admin/upgrade/status/health.go +++ b/pkg/cli/admin/upgrade/status/health.go @@ -95,10 +95,18 @@ type updateInsightImpact struct { description string } +func (i updateInsightImpact) incomplete() bool { + return i.description == "" || i.summary == "" +} + type updateInsightRemediation struct { reference string } +func (r updateInsightRemediation) incomplete() bool { + return r.reference == "" +} + type updateInsight struct { startedAt time.Time scope updateInsightScope @@ -106,6 +114,10 @@ type updateInsight struct { remediation updateInsightRemediation } +func (i updateInsight) incomplete() bool { + return i.impact.incomplete() || i.remediation.incomplete() +} + type updateHealthData struct { evaluatedAt time.Time insights []updateInsight @@ -113,7 +125,13 @@ type updateHealthData struct { func assessUpdateInsights(insights []updateInsight, upgradingFor time.Duration, evaluatedAt time.Time) updateHealthData { sorted := make([]updateInsight, 0, len(insights)) - sorted = append(sorted, insights...) + for _, insight := range insights { + if insight.incomplete() { + continue + } + sorted = append(sorted, insight) + } + sort.Slice(sorted, func(i, j int) bool { return sorted[i].startedAt.After(sorted[j].startedAt) }) diff --git a/pkg/cli/admin/upgrade/status/health_test.go b/pkg/cli/admin/upgrade/status/health_test.go index 07f6b4ba28..0f138e78b8 100644 --- a/pkg/cli/admin/upgrade/status/health_test.go +++ b/pkg/cli/admin/upgrade/status/health_test.go @@ -104,6 +104,65 @@ func TestAssessUpdateInsights_NoInsightsCreatesAllIsWellInfo(t *testing.T) { } } +func TestAssessUpdateInsights_FiltersOutIncompleteInsights(t *testing.T) { + assessedAt := now + var insights = []updateInsight{ + { + startedAt: now.Add(-20 * time.Second), + impact: updateInsightImpact{ + level: errorImpactLevel, + impactType: clusterCapacityImpactType, + // empty summary + summary: "", + description: "Autoscaler is disabled, you should enable it", + }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com/container-platform/4.14/nodes/pods/nodes-pods-autoscaling.html"}, + }, + { + startedAt: now.Add(-5 * time.Second), + impact: updateInsightImpact{ + level: errorImpactLevel, + impactType: apiAvailabilityImpactType, + summary: "Something that broke API and happened recently", + // empty description + description: "", + }, + remediation: updateInsightRemediation{reference: "https://docs.openshift.com/container-platform/4.14/authentication/understanding-authentication.html"}, + }, + { + startedAt: now.Add(-time.Hour), + impact: updateInsightImpact{ + level: criticalInfoLevel, + impactType: dataLossImpactType, + summary: "KTHXBAI DATA", + description: "You lost all your data, hope you have a backup", + }, + // empty reference + remediation: updateInsightRemediation{reference: ""}, + }, + } + + healthData := assessUpdateInsights(insights, 2*time.Hour, assessedAt) + // All incomplete insights are filtered out, so we only get the synthetic "All is well" insight + expected := updateHealthData{ + evaluatedAt: assessedAt, + insights: []updateInsight{ + { + startedAt: assessedAt.Add(-2 * time.Hour), + impact: updateInsightImpact{ + level: infoImpactLevel, + impactType: noneImpactType, + summary: "Upgrade is proceeding well", + }, + }, + }, + } + + if diff := cmp.Diff(expected, healthData, cmp.AllowUnexported(updateHealthData{}, updateInsight{}, updateInsightScope{}, updateInsightImpact{}, updateInsightRemediation{})); diff != "" { + t.Fatalf("Output differs from expected :\n%s", diff) + } +} + func TestUpdateHealthData_Write(t *testing.T) { testCases := []struct { name string From 77ecca9b59bc10c061d60c1936eb06e0ddee6bf2 Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Tue, 9 Apr 2024 16:27:45 +0200 Subject: [PATCH 05/10] `upgrade status`: minor fixes for node status --- .../examples/4.14.1-degraded.detailed-output | 6 +++++ .../4.14.1-paused-worker-pool.detailed-output | 10 ++++++-- .../examples/4.14.1-paused-worker-pool.output | 2 +- ...ed-updating-multiple-pools.detailed-output | 14 ++++++----- ...-ec2-b02-cos-not-annotated.detailed-output | 14 ++++++----- .../examples/4.15.0-ec2-early.detailed-output | 14 ++++++----- ....0-ec2-unavailable-mco-20m.detailed-output | 6 +++++ ...ted-pdb-prohibits-draining.detailed-output | 6 +++++ pkg/cli/admin/upgrade/status/health.go | 25 ++++++++++++------- pkg/cli/admin/upgrade/status/health_test.go | 21 +++++++++++++--- pkg/cli/admin/upgrade/status/status.go | 4 +-- pkg/cli/admin/upgrade/status/workerpool.go | 6 +++-- .../admin/upgrade/status/workerpool_test.go | 8 +++--- 13 files changed, 94 insertions(+), 42 deletions(-) diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output index 3cd50a1079..64696672ad 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output @@ -12,6 +12,12 @@ Completion: 97% Duration: 1h58m50s Operator Status: 33 Total, 32 Available, 1 Progressing, 4 Degraded +Control Plane Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? + = Worker Upgrade = = Worker Pool = diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output index c238a7a4c6..a7f507aa97 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output @@ -6,6 +6,12 @@ Completion: 12% Duration: 6s Operator Status: 33 Total, 33 Available, 0 Progressing, 0 Degraded +Control Plane Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0 ? + = Worker Upgrade = = Worker Pool = @@ -21,9 +27,9 @@ ip-10-0-4-159.us-east-2.compute.internal Excluded Paused 4.14.0 - ip-10-0-99-40.us-east-2.compute.internal Excluded Paused 4.14.0 - = Update Health = -Message: Worker pool worker is paused +Message: Outdated nodes in a paused pool 'worker' will not be updated Since: - Level: Warning Impact: Update Stalled Reference: https://docs.openshift.com/container-platform/4.15/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues - Description: Outdated nodes in a paused pool will not be updated + Description: Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator. diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output index 0689526a9f..efa519b55d 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output @@ -28,4 +28,4 @@ ip-10-0-99-40.us-east-2.compute.internal Excluded Paused 4.14.0 - = Update Health = SINCE LEVEL IMPACT MESSAGE -- Warning Update Stalled Worker pool worker is paused +- Warning Update Stalled Outdated nodes in a paused pool 'worker' will not be updated diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output index 901c06a79c..43639fb7c3 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output @@ -6,6 +6,12 @@ Completion: 97% Duration: 14m4s Operator Status: 33 Total, 32 Available, 1 Progressing, 0 Degraded +Control Plane Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-53-40.us-east-2.compute.internal Progressing Draining 4.14.0 +30m +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0 ? + = Worker Upgrade = = Worker Pool = @@ -32,9 +38,5 @@ ip-10-0-4-159-infra.us-east-2.compute.internal Progressing Draining 4.14. ip-10-0-20-162-infra.us-east-2.compute.internal Outdated Pending 4.14.0 ? = Update Health = -Message: Upgrade is proceeding well - Since: 14m4s - Level: Info - Impact: None - Reference: - Description: +SINCE LEVEL IMPACT MESSAGE +14m4s Info None Upgrade is proceeding well diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output index 49a42ff97c..5f3a9eb336 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output @@ -6,6 +6,12 @@ Completion: 43% Duration: 52m56s Operator Status: 7 Total, 7 Available, 0 Progressing, 0 Degraded +Control Plane Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? + = Worker Upgrade = = Worker Pool = @@ -21,9 +27,5 @@ ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? = Update Health = -Message: Upgrade is proceeding well - Since: 52m56s - Level: Info - Impact: None - Reference: - Description: +SINCE LEVEL IMPACT MESSAGE +52m56s Info None Upgrade is proceeding well diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output index 1d558de3cc..69afde8b0f 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output @@ -6,6 +6,12 @@ Completion: 3% Duration: 1m29s Operator Status: 33 Total, 33 Available, 2 Progressing, 0 Degraded +Control Plane Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.1 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.1 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.1 ? + = Worker Upgrade = = Worker Pool = @@ -21,9 +27,5 @@ ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.1 ? ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.1 ? = Update Health = -Message: Upgrade is proceeding well - Since: 1m29s - Level: Info - Impact: None - Reference: - Description: +SINCE LEVEL IMPACT MESSAGE +1m29s Info None Upgrade is proceeding well diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output index a5180b0cfd..44a792d344 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output @@ -6,6 +6,12 @@ Completion: 97% Duration: 58m53s Operator Status: 33 Total, 31 Available, 1 Progressing, 1 Degraded +Control Plane Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? + = Worker Upgrade = = Worker Pool = diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output index 95a9593161..4e96e452a4 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output @@ -12,6 +12,12 @@ Completion: 100% Duration: 3h30m31s Operator Status: 36 Total, 36 Available, 0 Progressing, 1 Degraded +Control Plane Node(s) +NAME ASSESSMENT PHASE VERSION EST MESSAGE +build0-gstfj-m-0.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-m-1.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-m-2.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - + = Worker Upgrade = = Worker Pool = diff --git a/pkg/cli/admin/upgrade/status/health.go b/pkg/cli/admin/upgrade/status/health.go index 8f4aa543ba..16fb038c83 100644 --- a/pkg/cli/admin/upgrade/status/health.go +++ b/pkg/cli/admin/upgrade/status/health.go @@ -123,7 +123,9 @@ type updateHealthData struct { insights []updateInsight } -func assessUpdateInsights(insights []updateInsight, upgradingFor time.Duration, evaluatedAt time.Time) updateHealthData { +// assessUpdateInsights processes insights to be displayed and returns matching displayable data. If the displayable data are not +// worth showing in detailed mode, returns false as the second return value. +func assessUpdateInsights(insights []updateInsight, upgradingFor time.Duration, evaluatedAt time.Time) (updateHealthData, bool) { sorted := make([]updateInsight, 0, len(insights)) for _, insight := range insights { if insight.incomplete() { @@ -140,20 +142,25 @@ func assessUpdateInsights(insights []updateInsight, upgradingFor time.Duration, }) if len(sorted) == 0 { - sorted = append(sorted, updateInsight{ - startedAt: evaluatedAt.Add(-upgradingFor), - impact: updateInsightImpact{ - level: infoImpactLevel, - impactType: noneImpactType, - summary: "Upgrade is proceeding well", + return updateHealthData{ + evaluatedAt: evaluatedAt, + insights: []updateInsight{ + { + startedAt: evaluatedAt.Add(-upgradingFor), + impact: updateInsightImpact{ + level: infoImpactLevel, + impactType: noneImpactType, + summary: "Upgrade is proceeding well", + }, + }, }, - }) + }, false } return updateHealthData{ evaluatedAt: evaluatedAt, insights: sorted, - } + }, true } func shortDuration(d time.Duration) string { diff --git a/pkg/cli/admin/upgrade/status/health_test.go b/pkg/cli/admin/upgrade/status/health_test.go index 0f138e78b8..ef58fd46f4 100644 --- a/pkg/cli/admin/upgrade/status/health_test.go +++ b/pkg/cli/admin/upgrade/status/health_test.go @@ -65,7 +65,7 @@ var insights = []updateInsight{ func TestAssessUpdateInsights_Sorts(t *testing.T) { assessedAt := now - healthData := assessUpdateInsights(insights, 2*time.Hour, assessedAt) + healthData, allowDetailed := assessUpdateInsights(insights, 2*time.Hour, assessedAt) var messages []string for _, insights := range healthData.insights { messages = append(messages, insights.impact.summary) @@ -77,6 +77,11 @@ func TestAssessUpdateInsights_Sorts(t *testing.T) { "Something that slows the update", "Something with no impact that happened right now", } + + if !allowDetailed { + t.Error("Non-artificial insights should be allowed for detailed output") + } + if diff := cmp.Diff(expected, messages); diff != "" { t.Fatalf("Output differs from expected :\n%s", diff) } @@ -84,7 +89,7 @@ func TestAssessUpdateInsights_Sorts(t *testing.T) { func TestAssessUpdateInsights_NoInsightsCreatesAllIsWellInfo(t *testing.T) { assessedAt := now - healthData := assessUpdateInsights(nil, 2*time.Hour, assessedAt) + healthData, allowDetailed := assessUpdateInsights(nil, 2*time.Hour, assessedAt) expected := updateHealthData{ evaluatedAt: assessedAt, insights: []updateInsight{ @@ -99,6 +104,10 @@ func TestAssessUpdateInsights_NoInsightsCreatesAllIsWellInfo(t *testing.T) { }, } + if allowDetailed { + t.Error("All is well insight should not be allowed for detailed output") + } + if diff := cmp.Diff(expected, healthData, cmp.AllowUnexported(updateHealthData{}, updateInsight{}, updateInsightScope{}, updateInsightImpact{}, updateInsightRemediation{})); diff != "" { t.Fatalf("Output differs from expected :\n%s", diff) } @@ -142,7 +151,7 @@ func TestAssessUpdateInsights_FiltersOutIncompleteInsights(t *testing.T) { }, } - healthData := assessUpdateInsights(insights, 2*time.Hour, assessedAt) + healthData, allowDetailed := assessUpdateInsights(insights, 2*time.Hour, assessedAt) // All incomplete insights are filtered out, so we only get the synthetic "All is well" insight expected := updateHealthData{ evaluatedAt: assessedAt, @@ -158,6 +167,10 @@ func TestAssessUpdateInsights_FiltersOutIncompleteInsights(t *testing.T) { }, } + if allowDetailed { + t.Error("All is well insight should not be allowed for detailed output") + } + if diff := cmp.Diff(expected, healthData, cmp.AllowUnexported(updateHealthData{}, updateInsight{}, updateInsightScope{}, updateInsightImpact{}, updateInsightRemediation{})); diff != "" { t.Fatalf("Output differs from expected :\n%s", diff) } @@ -194,7 +207,7 @@ func TestUpdateHealthData_Write(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var w strings.Builder - healthData := assessUpdateInsights(insights, 2*time.Hour, now) + healthData, _ := assessUpdateInsights(insights, 2*time.Hour, now) if err := healthData.Write(&w, tc.detailed); err != nil { t.Fatalf("Unexpected error: %v", err) } diff --git a/pkg/cli/admin/upgrade/status/status.go b/pkg/cli/admin/upgrade/status/status.go index c979a42fad..8e852e00b6 100644 --- a/pkg/cli/admin/upgrade/status/status.go +++ b/pkg/cli/admin/upgrade/status/status.go @@ -257,8 +257,8 @@ func (o *options) Run(ctx context.Context) error { } fmt.Fprintf(o.Out, "\n") - upgradeHealth := assessUpdateInsights(updateInsights, updatingFor, now) - _ = upgradeHealth.Write(o.Out, o.detailedOutput) + upgradeHealth, allowDetailed := assessUpdateInsights(updateInsights, updatingFor, now) + _ = upgradeHealth.Write(o.Out, o.detailedOutput && allowDetailed) return nil } diff --git a/pkg/cli/admin/upgrade/status/workerpool.go b/pkg/cli/admin/upgrade/status/workerpool.go index 281de1a63e..134279fc0c 100644 --- a/pkg/cli/admin/upgrade/status/workerpool.go +++ b/pkg/cli/admin/upgrade/status/workerpool.go @@ -431,6 +431,8 @@ func assessMachineConfigPool(pool mcfgv1.MachineConfigPool, nodes []nodeDisplayD } func machineConfigPoolInsights(poolDisplay poolDisplayData, pool mcfgv1.MachineConfigPool) (insights []updateInsight) { + // TODO: Only generate this insight if the pool has some work remaining that will not finish + // Depends on how MCO actually works: will it stop updating a node that already started e.g. draining?) if poolDisplay.NodesOverview.Excluded > 0 && pool.Spec.Paused { insights = append(insights, updateInsight{ startedAt: time.Time{}, @@ -441,8 +443,8 @@ func machineConfigPoolInsights(poolDisplay poolDisplayData, pool mcfgv1.MachineC impact: updateInsightImpact{ level: warningImpactLevel, impactType: updateStalledImpactType, - summary: fmt.Sprintf("Worker pool %s is paused", pool.Name), - description: "Outdated nodes in a paused pool will not be updated", + summary: fmt.Sprintf("Outdated nodes in a paused pool '%s' will not be updated", pool.Name), + description: "Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator.", }, remediation: updateInsightRemediation{ reference: "https://docs.openshift.com/container-platform/4.15/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues", diff --git a/pkg/cli/admin/upgrade/status/workerpool_test.go b/pkg/cli/admin/upgrade/status/workerpool_test.go index 6ae70ae501..befd826ff8 100644 --- a/pkg/cli/admin/upgrade/status/workerpool_test.go +++ b/pkg/cli/admin/upgrade/status/workerpool_test.go @@ -979,8 +979,8 @@ func Test_assessMachineConfigPool(t *testing.T) { impact: updateInsightImpact{ level: warningImpactLevel, impactType: updateStalledImpactType, - summary: "Worker pool worker is paused", - description: "Outdated nodes in a paused pool will not be updated", + summary: "Outdated nodes in a paused pool 'worker' will not be updated", + description: "Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator.", }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, @@ -1075,8 +1075,8 @@ func Test_machineConfigPoolInsights(t *testing.T) { impact: updateInsightImpact{ level: warningImpactLevel, impactType: updateStalledImpactType, - summary: "Worker pool worker is paused", - description: "Outdated nodes in a paused pool will not be updated", + summary: "Outdated nodes in a paused pool 'worker' will not be updated", + description: "Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator.", }, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, From 1d3922ca2b8530b8eea26774a1cf676191073427 Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Mon, 15 Apr 2024 19:48:54 +0200 Subject: [PATCH 06/10] squash: regenerate detailed outputs --- .../status/examples/4.14.1-degraded.detailed-output | 4 ++-- .../4.14.1-paused-worker-pool.detailed-output | 4 ++-- ...-started-updating-multiple-pools.detailed-output | 13 ++++++------- ...4.15.0-ec2-b02-cos-not-annotated.detailed-output | 4 ++-- .../examples/4.15.0-ec2-early.detailed-output | 4 ++-- .../4.15.0-ec2-unavailable-mco-20m.detailed-output | 4 ++-- ...e-updated-pdb-prohibits-draining.detailed-output | 4 ++-- 7 files changed, 18 insertions(+), 19 deletions(-) diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output index 64696672ad..33b9107de1 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output @@ -12,7 +12,7 @@ Completion: 97% Duration: 1h58m50s Operator Status: 33 Total, 32 Available, 1 Progressing, 4 Degraded -Control Plane Node(s) +Control Plane Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? @@ -26,7 +26,7 @@ Assessment: Pending Completion: 0% Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded -Worker Pool Node(s) +Worker Pool Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output index a7f507aa97..76c7df3871 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output @@ -6,7 +6,7 @@ Completion: 12% Duration: 6s Operator Status: 33 Total, 33 Available, 0 Progressing, 0 Degraded -Control Plane Node(s) +Control Plane Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0 ? ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0 ? @@ -20,7 +20,7 @@ Assessment: Excluded Completion: 0% Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 3 Excluded, 0 Degraded -Worker Pool Node(s) +Worker Pool Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-20-162.us-east-2.compute.internal Excluded Paused 4.14.0 - ip-10-0-4-159.us-east-2.compute.internal Excluded Paused 4.14.0 - diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output index 43639fb7c3..b63a6ea757 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output @@ -6,7 +6,7 @@ Completion: 97% Duration: 14m4s Operator Status: 33 Total, 32 Available, 1 Progressing, 0 Degraded -Control Plane Node(s) +Control Plane Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-53-40.us-east-2.compute.internal Progressing Draining 4.14.0 +30m ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0 ? @@ -20,7 +20,7 @@ Assessment: Progressing Completion: 0% Worker Status: 3 Total, 2 Available, 1 Progressing, 3 Outdated, 1 Draining, 0 Excluded, 0 Degraded -Worker Pool Node(s) +Worker Pool Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-4-159.us-east-2.compute.internal Progressing Draining 4.14.0 +30m ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.0 ? @@ -30,12 +30,11 @@ ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0 ? Worker Pool: infra Assessment: Progressing Completion: 0% -Worker Status: 2 Total, 1 Available, 1 Progressing, 2 Outdated, 1 Draining, 0 Excluded, 0 Degraded +Worker Status: 1 Total, 0 Available, 1 Progressing, 1 Outdated, 1 Draining, 0 Excluded, 0 Degraded -Worker Pool Node(s) -NAME ASSESSMENT PHASE VERSION EST MESSAGE -ip-10-0-4-159-infra.us-east-2.compute.internal Progressing Draining 4.14.0 +30m -ip-10-0-20-162-infra.us-east-2.compute.internal Outdated Pending 4.14.0 ? +Worker Pool Node +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-4-159-infra.us-east-2.compute.internal Progressing Draining 4.14.0 +30m = Update Health = SINCE LEVEL IMPACT MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output index 5f3a9eb336..b52d104618 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output @@ -6,7 +6,7 @@ Completion: 43% Duration: 52m56s Operator Status: 7 Total, 7 Available, 0 Progressing, 0 Degraded -Control Plane Node(s) +Control Plane Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? @@ -20,7 +20,7 @@ Assessment: Pending Completion: 0% Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded -Worker Pool Node(s) +Worker Pool Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output index 69afde8b0f..6567f43f52 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output @@ -6,7 +6,7 @@ Completion: 3% Duration: 1m29s Operator Status: 33 Total, 33 Available, 2 Progressing, 0 Degraded -Control Plane Node(s) +Control Plane Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.1 ? ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.1 ? @@ -20,7 +20,7 @@ Assessment: Pending Completion: 0% Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded -Worker Pool Node(s) +Worker Pool Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.1 ? ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.1 ? diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output index 44a792d344..0bcc9a2452 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output @@ -6,7 +6,7 @@ Completion: 97% Duration: 58m53s Operator Status: 33 Total, 31 Available, 1 Progressing, 1 Degraded -Control Plane Node(s) +Control Plane Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? @@ -20,7 +20,7 @@ Assessment: Pending Completion: 0% Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded -Worker Pool Node(s) +Worker Pool Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output index 4e96e452a4..3abc65b558 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output @@ -12,7 +12,7 @@ Completion: 100% Duration: 3h30m31s Operator Status: 36 Total, 36 Available, 0 Progressing, 1 Degraded -Control Plane Node(s) +Control Plane Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE build0-gstfj-m-0.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - build0-gstfj-m-1.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - @@ -26,7 +26,7 @@ Assessment: Degraded Completion: 39% Worker Status: 59 Total, 46 Available, 5 Progressing, 36 Outdated, 12 Draining, 0 Excluded, 7 Degraded -Worker Pool Node(s) +Worker Pool Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information From a79998876b4ab3cfa358d3013ed3182801320dab Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Mon, 15 Apr 2024 19:49:54 +0200 Subject: [PATCH 07/10] squash: ellipsize long node names in node section --- ...-updated-pdb-prohibits-draining.detailed-output | 14 +++++++------- ...rol-plane-updated-pdb-prohibits-draining.output | 14 +++++++------- pkg/cli/admin/upgrade/status/workerpool.go | 9 ++++++++- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output index 3abc65b558..3fd4747e77 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output @@ -28,13 +28,13 @@ Worker Status: 59 Total, 46 Available, 5 Progressing, 36 Outdated, 12 Draining Worker Pool Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE -build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information build0-gstfj-ci-tests-worker-c-jq5rk Unavailable Updated 4.16.0-ec.3 - Node is unavailable build0-gstfj-ci-tests-worker-c-2kz4m Progressing Draining 4.16.0-ec.2 +30m build0-gstfj-ci-tests-worker-c-55hpj Progressing Draining 4.16.0-ec.2 +30m diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output index 18da08305d..aefa740504 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output @@ -28,13 +28,13 @@ Worker Status: 59 Total, 46 Available, 5 Progressing, 36 Outdated, 12 Draining Worker Pool Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE -build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information build0-gstfj-ci-tests-worker-c-jq5rk Unavailable Updated 4.16.0-ec.3 - Node is unavailable build0-gstfj-ci-tests-worker-c-2kz4m Progressing Draining 4.16.0-ec.2 +30m build0-gstfj-ci-tests-worker-c-55hpj Progressing Draining 4.16.0-ec.2 +30m diff --git a/pkg/cli/admin/upgrade/status/workerpool.go b/pkg/cli/admin/upgrade/status/workerpool.go index 134279fc0c..989474a12b 100644 --- a/pkg/cli/admin/upgrade/status/workerpool.go +++ b/pkg/cli/admin/upgrade/status/workerpool.go @@ -198,7 +198,7 @@ func assessNodesStatus(cv *configv1.ClusterVersion, pool mcfgv1.MachineConfigPoo Assessment: assessment, Estimate: estimate, Phase: phase, - Message: message, + Message: strings.Replace(message, node.Name, ellipsize(node.Name), -1), Version: currentVersion, isUnavailable: isUnavailable, isDegraded: isDegraded, @@ -220,6 +220,13 @@ func assessNodesStatus(cv *configv1.ClusterVersion, pool mcfgv1.MachineConfigPoo return nodesStatusData, insights } +func ellipsize(name string) string { + if len(name) <= 10 { + return name + } + return "" +} + func getOpenShiftVersionOfMachineConfig(machineConfigs []mcfgv1.MachineConfig, name string) string { for _, mc := range machineConfigs { if mc.Name == name { From 37f440909568c6d8172088bc919cf8b27214dc5d Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Tue, 16 Apr 2024 14:02:11 +0200 Subject: [PATCH 08/10] `update status`: coalesce similar degraded node insights --- .../admin/upgrade/status/controlplane_test.go | 10 +- ...ted-pdb-prohibits-draining.detailed-output | 46 +---- ...lane-updated-pdb-prohibits-draining.output | 8 +- pkg/cli/admin/upgrade/status/workerpool.go | 184 +++++++++++++++--- .../admin/upgrade/status/workerpool_test.go | 85 ++++---- 5 files changed, 208 insertions(+), 125 deletions(-) diff --git a/pkg/cli/admin/upgrade/status/controlplane_test.go b/pkg/cli/admin/upgrade/status/controlplane_test.go index c80f3035de..dc5a6d4db1 100644 --- a/pkg/cli/admin/upgrade/status/controlplane_test.go +++ b/pkg/cli/admin/upgrade/status/controlplane_test.go @@ -129,7 +129,15 @@ var cvFixture = configv1.ClusterVersion{ }, } -var allowUnexportedInsightStructs = cmp.AllowUnexported(updateInsight{}, updateInsightScope{}, scopeResource{}, updateInsightImpact{}, updateInsightRemediation{}) +var allowUnexportedInsightStructs = cmp.AllowUnexported( + updateInsight{}, + updateInsightScope{}, + scopeResource{}, + updateInsightImpact{}, + updateInsightRemediation{}, + unavailableNodesInsight{}, + degradedNodesInsight{}, +) func TestAssessControlPlaneStatus_Operators(t *testing.T) { testCases := []struct { diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output index 3fd4747e77..f08c277fe7 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output @@ -43,54 +43,12 @@ Omitted additional 49 Total, 22 Completed, 46 Available, 3 Progressing, 27 Outda Pass along --details to see all information. = Update Health = -Message: Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded +Message: 7 nodes in pool worker are degraded while draining: failed to drain node: after 1 hour. Please see machine-config-controller logs for more information Since: - Level: Error Impact: Update Stalled Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator - Description: failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information - -Message: Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded - Since: - - Level: Error - Impact: Update Stalled - Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator - Description: failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information - -Message: Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded - Since: - - Level: Error - Impact: Update Stalled - Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator - Description: failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information - -Message: Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded - Since: - - Level: Error - Impact: Update Stalled - Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator - Description: failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information - -Message: Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded - Since: - - Level: Error - Impact: Update Stalled - Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator - Description: failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information - -Message: Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded - Since: - - Level: Error - Impact: Update Stalled - Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator - Description: failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information - -Message: Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded - Since: - - Level: Error - Impact: Update Stalled - Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator - Description: failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information + Description: failed to drain node: after 1 hour. Please see machine-config-controller logs for more information Message: Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable Since: - diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output index aefa740504..4b393b7141 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output @@ -44,11 +44,5 @@ Pass along --details to see all information. = Update Health = SINCE LEVEL IMPACT MESSAGE -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded -- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded -- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded -- Error Update Stalled Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded +- Error Update Stalled 7 nodes in pool worker are degraded while draining: failed to drain node: after 1 hour. Please see machine-config-controller logs for more information - Warning Update Speed Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable diff --git a/pkg/cli/admin/upgrade/status/workerpool.go b/pkg/cli/admin/upgrade/status/workerpool.go index 989474a12b..f1a5a049ff 100644 --- a/pkg/cli/admin/upgrade/status/workerpool.go +++ b/pkg/cli/admin/upgrade/status/workerpool.go @@ -1,9 +1,11 @@ package status import ( + "cmp" "context" "fmt" "io" + "slices" "sort" "strings" "text/tabwriter" @@ -13,6 +15,7 @@ import ( corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" configv1 "github.com/openshift/api/config/v1" mcfgv1 "github.com/openshift/api/machineconfiguration/v1" @@ -135,9 +138,112 @@ func selectNodesFromPool(pool mcfgv1.MachineConfigPool, allNodes []corev1.Node) return res, nil } +type multipleNodeInsight interface { + merge(insight multipleNodeInsight) bool + render() updateInsight +} + +type unavailableNodesInsight struct { + // TODO(muller): Unwrap this + insight updateInsight +} + +func (a *unavailableNodesInsight) merge(_ multipleNodeInsight) bool { + return false +} + +func (a *unavailableNodesInsight) render() updateInsight { + return a.insight +} + +type degradedNodesInsight struct { + pool string + phase nodePhase + startedAt time.Time + scope updateInsightScope + description string +} + +func (d *degradedNodesInsight) merge(insight multipleNodeInsight) bool { + other, ok := insight.(*degradedNodesInsight) + if !ok { + return false + } + + if d.scope.scopeType != other.scope.scopeType { + return false + } + + myDesc := d.description + if len(d.scope.resources) == 1 { + nodeName := d.scope.resources[0].name + myDesc = strings.Replace(myDesc, nodeName, ellipsize(nodeName, 5), -1) + } + otherDesc := other.description + if len(other.scope.resources) == 1 { + nodeName := other.scope.resources[0].name + otherDesc = strings.Replace(otherDesc, nodeName, ellipsize(nodeName, 5), -1) + } + + if myDesc != otherDesc { + return false + } + d.description = myDesc + d.scope.resources = append(d.scope.resources, other.scope.resources...) + + if other.startedAt.Before(d.startedAt) { + d.startedAt = other.startedAt + } + + return true +} + +func (d *degradedNodesInsight) render() updateInsight { + // Deduplicate and sort resources + resources := sets.New[scopeResource](d.scope.resources...) + d.scope.resources = resources.UnsortedList() + slices.SortFunc(d.scope.resources, func(a, b scopeResource) int { + if n := cmp.Compare(a.kind, b.kind); n != 0 { + return n + } + if n := cmp.Compare(a.namespace, b.namespace); n != 0 { + return n + } + return cmp.Compare(a.name, b.name) + }) + + var node uint + for _, resource := range d.scope.resources { + if resource.kind == scopeKindNode { + node += 1 + } + } + + var summary string + if nodes := len(d.scope.resources); nodes > 1 { + summary = fmt.Sprintf("%d nodes in pool %s are degraded while %s: %s", nodes, d.pool, strings.ToLower(d.phase.String()), d.description) + } else { + summary = fmt.Sprintf("Node %s in pool %s is degraded while %s: %s", d.scope.resources[0].name, d.pool, strings.ToLower(d.phase.String()), d.description) + } + + return updateInsight{ + startedAt: d.startedAt, + scope: d.scope, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, + impact: updateInsightImpact{ + level: errorImpactLevel, + impactType: updateStalledImpactType, + summary: summary, + description: d.description, + }, + } +} + func assessNodesStatus(cv *configv1.ClusterVersion, pool mcfgv1.MachineConfigPool, nodes []corev1.Node, machineConfigs []mcfgv1.MachineConfig) ([]nodeDisplayData, []updateInsight) { var nodesStatusData []nodeDisplayData - var insights []updateInsight + var multipleNodeInsights []multipleNodeInsight for _, node := range nodes { currentVersion := getOpenShiftVersionOfMachineConfig(machineConfigs, node.Annotations[mco.CurrentMachineConfigAnnotationKey]) desiredVersion := getOpenShiftVersionOfMachineConfig(machineConfigs, node.Annotations[mco.DesiredMachineConfigAnnotationKey]) @@ -191,14 +297,27 @@ func assessNodesStatus(cv *configv1.ClusterVersion, pool mcfgv1.MachineConfigPoo } } - insights = append(insights, nodeInsights(pool, node, message, isUnavailable, isUpdating, isDegraded)...) + singleNodeInsights := nodeInsights(pool, node, message, phase, isUnavailable, isUpdating, isDegraded) + for si := range singleNodeInsights { + merged := false + insight := singleNodeInsights[si] + for mi := range multipleNodeInsights { + if ok := multipleNodeInsights[mi].merge(insight); ok { + merged = true + break + } + } + if !merged { + multipleNodeInsights = append(multipleNodeInsights, insight) + } + } nodesStatusData = append(nodesStatusData, nodeDisplayData{ Name: node.Name, Assessment: assessment, Estimate: estimate, Phase: phase, - Message: strings.Replace(message, node.Name, ellipsize(node.Name), -1), + Message: strings.Replace(message, node.Name, ellipsize(node.Name, 10), -1), Version: currentVersion, isUnavailable: isUnavailable, isDegraded: isDegraded, @@ -217,11 +336,18 @@ func assessNodesStatus(cv *configv1.ClusterVersion, pool mcfgv1.MachineConfigPoo return nodesStatusData[i].Assessment < nodesStatusData[j].Assessment }) + var insights []updateInsight + if len(multipleNodeInsights) > 0 { + insights = make([]updateInsight, 0, len(multipleNodeInsights)) + for i := range multipleNodeInsights { + insights = append(insights, multipleNodeInsights[i].render()) + } + } return nodesStatusData, insights } -func ellipsize(name string) string { - if len(name) <= 10 { +func ellipsize(name string, aboveLen int) string { + if len(name) <= aboveLen { return name } return "" @@ -325,46 +451,42 @@ func mcdUpdatingStateToPhase(state string) nodePhase { } } -func nodeInsights(pool mcfgv1.MachineConfigPool, node corev1.Node, reason string, isUnavailable, isUpdating, isDegraded bool) []updateInsight { - var insights []updateInsight +func nodeInsights(pool mcfgv1.MachineConfigPool, node corev1.Node, reason string, phase nodePhase, isUnavailable, isUpdating, isDegraded bool) []multipleNodeInsight { + var insights []multipleNodeInsight scope := scopeTypeWorkerPool if pool.Name == "master" { scope = scopeTypeControlPlane } if isUnavailable && !isUpdating { - insights = append(insights, updateInsight{ - startedAt: time.Time{}, - scope: updateInsightScope{ - scopeType: scope, - resources: []scopeResource{{kind: scopeKindNode, name: node.Name}}, - }, - impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: fmt.Sprintf("Node %s is unavailable", node.Name), - description: reason, - }, - remediation: updateInsightRemediation{ - reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + insights = append(insights, &unavailableNodesInsight{ + insight: updateInsight{ + startedAt: time.Time{}, + scope: updateInsightScope{ + scopeType: scope, + resources: []scopeResource{{kind: scopeKindNode, name: node.Name}}, + }, + impact: updateInsightImpact{ + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: fmt.Sprintf("Node %s is unavailable", node.Name), + description: reason, + }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }) } if isDegraded { - insights = append(insights, updateInsight{ + insights = append(insights, °radedNodesInsight{ + pool: pool.Name, + phase: phase, startedAt: time.Time{}, scope: updateInsightScope{ scopeType: scope, resources: []scopeResource{{kind: scopeKindNode, name: node.Name}}, }, - impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: updateStalledImpactType, - summary: fmt.Sprintf("Node %s is degraded", node.Name), - description: reason, - }, - remediation: updateInsightRemediation{ - reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", - }, + description: reason, }) } return insights diff --git a/pkg/cli/admin/upgrade/status/workerpool_test.go b/pkg/cli/admin/upgrade/status/workerpool_test.go index befd826ff8..861760d9ad 100644 --- a/pkg/cli/admin/upgrade/status/workerpool_test.go +++ b/pkg/cli/admin/upgrade/status/workerpool_test.go @@ -466,7 +466,7 @@ func Test_assessNodesStatus(t *testing.T) { impact: updateInsightImpact{ level: errorImpactLevel, impactType: updateStalledImpactType, - summary: "Node a is degraded", + summary: "Node a in pool master is degraded while draining: PDB prohibits draining", description: "PDB prohibits draining", }, scope: updateInsightScope{ @@ -657,6 +657,7 @@ func Test_nodeInsights(t *testing.T) { pool mcfgv1.MachineConfigPool node corev1.Node reason string + phase nodePhase isUnavailable bool isUpdating bool isDegraded bool @@ -664,7 +665,7 @@ func Test_nodeInsights(t *testing.T) { testCases := []struct { name string args args - expectedUpdateInsight []updateInsight + expectedUpdateInsight []multipleNodeInsight }{ { name: "node is updated - all is well", @@ -680,22 +681,25 @@ func Test_nodeInsights(t *testing.T) { pool: mcpMaster, node: node("a").updated(mcNew.Name, mcNew.Name).unavailable().node, reason: "Node is unavailable", + phase: phaseStateUpdated, isUnavailable: true, }, - expectedUpdateInsight: []updateInsight{ - { - impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Node a is unavailable", - description: "Node is unavailable", - }, - scope: updateInsightScope{ - scopeType: scopeTypeControlPlane, - resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, - }, - remediation: updateInsightRemediation{ - reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + expectedUpdateInsight: []multipleNodeInsight{ + &unavailableNodesInsight{ + updateInsight{ + impact: updateInsightImpact{ + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Node a is unavailable", + description: "Node is unavailable", + }, + scope: updateInsightScope{ + scopeType: scopeTypeControlPlane, + resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, + }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }, }, @@ -708,20 +712,22 @@ func Test_nodeInsights(t *testing.T) { reason: "Node is unavailable", isUnavailable: true, }, - expectedUpdateInsight: []updateInsight{ - { - impact: updateInsightImpact{ - level: warningImpactLevel, - impactType: updateSpeedImpactType, - summary: "Node a is unavailable", - description: "Node is unavailable", - }, - scope: updateInsightScope{ - scopeType: scopeTypeWorkerPool, - resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, - }, - remediation: updateInsightRemediation{ - reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + expectedUpdateInsight: []multipleNodeInsight{ + &unavailableNodesInsight{ + updateInsight{ + impact: updateInsightImpact{ + level: warningImpactLevel, + impactType: updateSpeedImpactType, + summary: "Node a is unavailable", + description: "Node is unavailable", + }, + scope: updateInsightScope{ + scopeType: scopeTypeWorkerPool, + resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, + }, + remediation: updateInsightRemediation{ + reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", + }, }, }, }, @@ -732,32 +738,27 @@ func Test_nodeInsights(t *testing.T) { pool: mcpWorker, node: node("a").degraded_draining(mcOld.Name, mcNew.Name, "PDB prohibits draining").node, reason: "PDB prohibits draining", + phase: phaseStateDraining, isUnavailable: true, isUpdating: true, isDegraded: true, }, - expectedUpdateInsight: []updateInsight{ - { - impact: updateInsightImpact{ - level: errorImpactLevel, - impactType: updateStalledImpactType, - summary: "Node a is degraded", - description: "PDB prohibits draining", - }, + expectedUpdateInsight: []multipleNodeInsight{ + °radedNodesInsight{ + pool: mcpWorker.Name, + phase: phaseStateDraining, scope: updateInsightScope{ scopeType: scopeTypeWorkerPool, resources: []scopeResource{{kind: scopeKindNode, name: "a"}}, }, - remediation: updateInsightRemediation{ - reference: "https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator", - }, + description: "PDB prohibits draining", }, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - updateInsights := nodeInsights(tc.args.pool, tc.args.node, tc.args.reason, tc.args.isUnavailable, tc.args.isUpdating, tc.args.isDegraded) + updateInsights := nodeInsights(tc.args.pool, tc.args.node, tc.args.reason, tc.args.phase, tc.args.isUnavailable, tc.args.isUpdating, tc.args.isDegraded) if diff := cmp.Diff(tc.expectedUpdateInsight, updateInsights, allowUnexportedInsightStructs); diff != "" { t.Errorf("updateInsight differ from expected:\n%s", diff) } From 40db17feb9bbce4a5a27d520289d9e9c97808866 Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Tue, 16 Apr 2024 19:26:18 +0200 Subject: [PATCH 09/10] `upgrade status`: emit resources in detailed mode --- .../examples/4.14.1-degraded.detailed-output | 10 +++++ .../4.14.1-paused-worker-pool.detailed-output | 2 + ....0-ec2-unavailable-mco-20m.detailed-output | 2 + ...ted-pdb-prohibits-draining.detailed-output | 4 ++ pkg/cli/admin/upgrade/status/health.go | 39 +++++++++++++++++++ 5 files changed, 57 insertions(+) diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output index 33b9107de1..65ac2d42de 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output @@ -38,6 +38,8 @@ Message: Cluster Operator kube-apiserver is degraded (NodeController_MasterNodes Level: Error Impact: API Availability Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Resources: + ClusterOperators: kube-apiserver Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) Message: Cluster Operator kube-controller-manager is degraded (NodeController_MasterNodesReady) @@ -45,6 +47,8 @@ Message: Cluster Operator kube-controller-manager is degraded (NodeController_Ma Level: Error Impact: API Availability Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Resources: + ClusterOperators: kube-controller-manager Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) Message: Cluster Operator kube-scheduler is degraded (NodeController_MasterNodesReady) @@ -52,6 +56,8 @@ Message: Cluster Operator kube-scheduler is degraded (NodeController_MasterNodes Level: Error Impact: API Availability Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Resources: + ClusterOperators: kube-scheduler Description: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) Message: Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady) @@ -59,6 +65,8 @@ Message: Cluster Operator etcd is degraded (EtcdEndpoints_ErrorUpdatingEtcdEndpo Level: Error Impact: API Availability Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDegraded.md + Resources: + ClusterOperators: etcd Description: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] , EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy , NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) @@ -68,4 +76,6 @@ Message: Cluster Operator control-plane-machine-set is unavailable (UnavailableR Level: Error Impact: API Availability Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md + Resources: + ClusterOperators: control-plane-machine-set Description: Missing 1 available replica(s) diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output index 76c7df3871..68a4c420b7 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output @@ -32,4 +32,6 @@ Message: Outdated nodes in a paused pool 'worker' will not be updated Level: Warning Impact: Update Stalled Reference: https://docs.openshift.com/container-platform/4.15/support/troubleshooting/troubleshooting-operator-issues.html#troubleshooting-disabling-autoreboot-mco_troubleshooting-operator-issues + Resources: + MachineConfigPools: worker Description: Pool is paused, which stops all changes to the nodes in the pool, including updates. The nodes will not be updated until the pool is unpaused by the administrator. diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output index 0bcc9a2452..b8fb80428f 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output @@ -32,4 +32,6 @@ Message: Cluster Operator machine-config is unavailable (MachineConfigController Level: Error Impact: API Availability Reference: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/ClusterOperatorDown.md + Resources: + ClusterOperators: machine-config Description: Cluster not available for [{operator 4.14.0-rc.3}]: ControllerConfig.machineconfiguration.openshift.io "machine-config-controller" is invalid: [status.controllerCertificates[0].notAfter: Required value, status.controllerCertificates[0].notBefore: Required value, status.controllerCertificates[1].notAfter: Required value, status.controllerCertificates[1].notBefore: Required value, status.controllerCertificates[2].notAfter: Required value, status.controllerCertificates[2].notBefore: Required value, status.controllerCertificates[3].notAfter: Required value, status.controllerCertificates[3].notBefore: Required value, status.controllerCertificates[4].notAfter: Required value, status.controllerCertificates[4].notBefore: Required value, status.controllerCertificates[5].notAfter: Required value, status.controllerCertificates[5].notBefore: Required value, status.controllerCertificates[6].notAfter: Required value, status.controllerCertificates[6].notBefore: Required value, status.controllerCertificates[7].notAfter: Required value, status.controllerCertificates[7].notBefore: Required value, status.controllerCertificates[8].notAfter: Required value, status.controllerCertificates[8].notBefore: Required value, status.controllerCertificates[9].notAfter: Required value, status.controllerCertificates[9].notBefore: Required value, : Invalid value: "null": some validation rules were not checked because the object was invalid; correct the existing errors to complete validation] diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output index f08c277fe7..37cdade6f9 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output @@ -48,6 +48,8 @@ Message: 7 nodes in pool worker are degraded while draining: failed to drain nod Level: Error Impact: Update Stalled Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Resources: + Nodes: build0-gstfj-ci-prowjobs-worker-b-9lztv build0-gstfj-ci-prowjobs-worker-b-bg9f5 build0-gstfj-ci-prowjobs-worker-b-mrxwn build0-gstfj-ci-tests-worker-b-4h7pn build0-gstfj-ci-tests-worker-b-jv5bg build0-gstfj-ci-tests-worker-b-kj6gk build0-gstfj-ci-tests-worker-c-dcz9p Description: failed to drain node: after 1 hour. Please see machine-config-controller logs for more information Message: Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable @@ -55,4 +57,6 @@ Message: Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable Level: Warning Impact: Update Speed Reference: https://docs.openshift.com/container-platform/4.15/post_installation_configuration/machine-configuration-tasks.html#understanding-the-machine-config-operator + Resources: + Nodes: build0-gstfj-ci-tests-worker-c-jq5rk Description: Node is unavailable diff --git a/pkg/cli/admin/upgrade/status/health.go b/pkg/cli/admin/upgrade/status/health.go index 16fb038c83..7ab2d1294a 100644 --- a/pkg/cli/admin/upgrade/status/health.go +++ b/pkg/cli/admin/upgrade/status/health.go @@ -7,6 +7,8 @@ import ( "strings" "text/tabwriter" "time" + + "k8s.io/apimachinery/pkg/util/sets" ) type scopeType string @@ -30,6 +32,14 @@ type scopeResource struct { name string } +func (r scopeResource) namespacedName() string { + if r.namespace == "" { + return r.name + } else { + return fmt.Sprintf("%s/%s", r.namespace, r.name) + } +} + type updateInsightScope struct { scopeType scopeType resources []scopeResource @@ -194,6 +204,7 @@ type displayItem struct { message string description string reference string + resources []scopeResource } func (i *updateHealthData) Write(w io.Writer, detailed bool) error { @@ -209,6 +220,7 @@ func (i *updateHealthData) Write(w io.Writer, detailed bool) error { message: insight.impact.summary, description: insight.impact.description, reference: insight.remediation.reference, + resources: insight.scope.resources, }) } if detailed { @@ -220,6 +232,31 @@ func (i *updateHealthData) Write(w io.Writer, detailed bool) error { return nil } +func detailedResourceOutput(w io.Writer, resources []scopeResource) { + if len(resources) == 0 { + return + } + _, _ = w.Write([]byte(fmt.Sprintf(" %s\n", "Resources:"))) + var pad int + byKind := make(map[allowedScopeKind][]string) + + kinds := sets.New[allowedScopeKind]() + for _, resource := range resources { + byKind[resource.kind] = append(byKind[resource.kind], resource.namespacedName()) + kinds.Insert(resource.kind) + if n := len(resource.kind); n > pad { + pad = n + } + } + + for _, kind := range sets.List(kinds) { + _, _ = w.Write([]byte(fmt.Sprintf(" %-*ss: ", pad, kind))) + sort.Strings(byKind[kind]) + _, _ = w.Write([]byte(strings.Join(byKind[kind], " ") + "\n")) + } + +} + func detailedOutput(w io.Writer, items []displayItem) { pad := len("Description: ") for i, item := range items { @@ -228,6 +265,8 @@ func detailedOutput(w io.Writer, items []displayItem) { _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Level:", item.level))) _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Impact:", item.impact))) _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Reference:", item.reference))) + + detailedResourceOutput(w, item.resources) // Respect the " Description: " indentation when description has linebreaks item.description = strings.ReplaceAll(item.description, "\n", fmt.Sprintf("\n%s, ", strings.Repeat(" ", pad+2))) _, _ = w.Write([]byte(fmt.Sprintf(" %-*s%s\n", pad, "Description:", item.description))) From 6da69b9f47c335062a93d7e1e842f8abc99bcd6b Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Tue, 16 Apr 2024 20:02:16 +0200 Subject: [PATCH 10/10] `upgrade status`: separate detailed mode for nodes and health --- ...ted-pdb-prohibits-draining.detailed-output | 74 +++++++++++++++---- ...lane-updated-pdb-prohibits-draining.output | 2 +- pkg/cli/admin/upgrade/status/examples_test.go | 6 +- pkg/cli/admin/upgrade/status/status.go | 32 ++++++-- pkg/cli/admin/upgrade/status/workerpool.go | 6 +- 5 files changed, 94 insertions(+), 26 deletions(-) diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output index 37cdade6f9..f77435b617 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output @@ -27,20 +27,66 @@ Completion: 39% Worker Status: 59 Total, 46 Available, 5 Progressing, 36 Outdated, 12 Draining, 0 Excluded, 7 Degraded Worker Pool Nodes -NAME ASSESSMENT PHASE VERSION EST MESSAGE -build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information -build0-gstfj-ci-tests-worker-c-jq5rk Unavailable Updated 4.16.0-ec.3 - Node is unavailable -build0-gstfj-ci-tests-worker-c-2kz4m Progressing Draining 4.16.0-ec.2 +30m -build0-gstfj-ci-tests-worker-c-55hpj Progressing Draining 4.16.0-ec.2 +30m -... -Omitted additional 49 Total, 22 Completed, 46 Available, 3 Progressing, 27 Outdated, 3 Draining, 0 Excluded, and 0 Degraded nodes. -Pass along --details to see all information. +NAME ASSESSMENT PHASE VERSION EST MESSAGE +build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-c-jq5rk Unavailable Updated 4.16.0-ec.3 - Node is unavailable +build0-gstfj-ci-tests-worker-c-2kz4m Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-tests-worker-c-55hpj Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-tests-worker-c-8tmjv Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-tests-worker-c-lfz8m Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-tests-worker-c-r55wv Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-longtests-worker-c-vspm9 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-longtests-worker-d-hksc9 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-c-f6dkk Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-c-kkm72 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-c-p5df7 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-c-tgrsc Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-d-ddnxd Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-2c4vg Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-5twk4 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-65n48 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-79nrd Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-82z85 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-glhn7 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-hnxl9 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-jpzfm Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-lxqgs Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-rqch4 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-zd9c7 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-6hddk Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-kc6xt Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-r2k42 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-r9zqc Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-w2bgd Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-w-d-7rj9w.c.openshift-ci-build-farm.internal Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-builds-worker-b-65df4 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-b-dvrd2 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-b-f7rlh Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-b-m4vgr Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-b-mg8bd Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-c-2hzgm Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-c-dmjft Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-c-ng8s5 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-c-vmmfj Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-d-4pjlz Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-d-z49wm Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-tests-worker-b-d9vz2 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-tests-worker-b-fkn28 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-tests-worker-b-mc7fm Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-tests-worker-b-pqz8t Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-f7dkq.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-infra-2stfh.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-infra-mhb8g.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-infra-n9kc5.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-mwf8t.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-c-nsmn5.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-c-qjbdl.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - = Update Health = Message: 7 nodes in pool worker are degraded while draining: failed to drain node: after 1 hour. Please see machine-config-controller logs for more information diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output index 4b393b7141..c5ce4de1ce 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output @@ -40,7 +40,7 @@ build0-gstfj-ci-tests-worker-c-2kz4m Progressing Draining 4.16.0-ec.2 build0-gstfj-ci-tests-worker-c-55hpj Progressing Draining 4.16.0-ec.2 +30m ... Omitted additional 49 Total, 22 Completed, 46 Available, 3 Progressing, 27 Outdated, 3 Draining, 0 Excluded, and 0 Degraded nodes. -Pass along --details to see all information. +Pass along --details=nodes to see all information. = Update Health = SINCE LEVEL IMPACT MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples_test.go b/pkg/cli/admin/upgrade/status/examples_test.go index bf1e7ef4db..1ba2cce7bf 100644 --- a/pkg/cli/admin/upgrade/status/examples_test.go +++ b/pkg/cli/admin/upgrade/status/examples_test.go @@ -45,17 +45,17 @@ func TestExamples(t *testing.T) { variants := []struct { name string - detailed bool + detailed string outputSuffix string }{ { name: "normal output", - detailed: false, + detailed: "none", outputSuffix: ".output", }, { name: "detailed output", - detailed: true, + detailed: "all", outputSuffix: ".detailed-output", }, } diff --git a/pkg/cli/admin/upgrade/status/status.go b/pkg/cli/admin/upgrade/status/status.go index 8e852e00b6..8fa5a0aaca 100644 --- a/pkg/cli/admin/upgrade/status/status.go +++ b/pkg/cli/admin/upgrade/status/status.go @@ -11,6 +11,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/cli-runtime/pkg/genericiooptions" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" kcmdutil "k8s.io/kubectl/pkg/cmd/util" @@ -34,6 +35,23 @@ func newOptions(streams genericiooptions.IOStreams) *options { } } +const ( + detailedOutputNone = "none" + detailedOutputNodes = "nodes" + detailedOutputHealth = "health" + detailedOutputAll = "all" +) + +func detailedHealthEnabled(detailedOutput string) bool { + return detailedOutput == detailedOutputHealth || detailedOutput == detailedOutputAll +} + +func detailedNodesEnabled(detailedOutput string) bool { + return detailedOutput == detailedOutputNodes || detailedOutput == detailedOutputAll +} + +var allDetailedOutputValues = sets.New[string](detailedOutputNone, detailedOutputNodes, detailedOutputHealth, detailedOutputAll) + func New(f kcmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Command { o := newOptions(streams) cmd := &cobra.Command{ @@ -49,7 +67,7 @@ func New(f kcmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Command // TODO: We can remove these flags once the idea about `oc adm upgrade status` stabilizes and the command // is promoted out of the OC_ENABLE_CMD_UPGRADE_STATUS feature gate flags.StringVar(&o.mockData.cvPath, "mock-clusterversion", "", "Path to a YAML ClusterVersion object to use for testing (will be removed later). Files in the same directory with the same name and suffixes -co.yaml, -mcp.yaml, -mc.yaml, and -node.yaml are required.") - flags.BoolVar(&o.detailedOutput, "details", false, "Show upgrade health insights with detailed information and remediating links") + flags.StringVar(&o.detailedOutput, "details", detailedOutputNone, fmt.Sprintf("Show more detailed output in some sections. One of %s", strings.Join(sets.List(allDetailedOutputValues), ", "))) return cmd } @@ -63,7 +81,7 @@ type options struct { CoreClient corev1client.CoreV1Interface MachineConfigClient machineconfigv1client.Interface - detailedOutput bool + detailedOutput string } func (o *options) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { @@ -71,6 +89,10 @@ func (o *options) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string return kcmdutil.UsageErrorf(cmd, "positional arguments given") } + if !allDetailedOutputValues.Has(o.detailedOutput) { + return kcmdutil.UsageErrorf(cmd, "invalid value for --details: %s (must be one of: %s)", o.detailedOutput, strings.Join(allDetailedOutputValues.UnsortedList(), ", ")) + } + cvSuffix := "-cv.yaml" if o.mockData.cvPath != "" { o.mockData.operatorsPath = strings.Replace(o.mockData.cvPath, cvSuffix, "-co.yaml", 1) @@ -247,18 +269,18 @@ func (o *options) Run(ctx context.Context) error { updateInsights = append(updateInsights, insights...) fmt.Fprintf(o.Out, "\n") _ = controlPlaneStatusData.Write(o.Out) - controlPlanePoolStatusData.WriteNodes(o.Out) + controlPlanePoolStatusData.WriteNodes(o.Out, detailedNodesEnabled(o.detailedOutput)) fmt.Fprintf(o.Out, "\n= Worker Upgrade =\n") for _, pool := range workerPoolsStatusData { fmt.Fprintf(o.Out, "\n") _ = pool.WritePool(o.Out) - pool.WriteNodes(o.Out) + pool.WriteNodes(o.Out, detailedNodesEnabled(o.detailedOutput)) } fmt.Fprintf(o.Out, "\n") upgradeHealth, allowDetailed := assessUpdateInsights(updateInsights, updatingFor, now) - _ = upgradeHealth.Write(o.Out, o.detailedOutput && allowDetailed) + _ = upgradeHealth.Write(o.Out, detailedHealthEnabled(o.detailedOutput) && allowDetailed) return nil } diff --git a/pkg/cli/admin/upgrade/status/workerpool.go b/pkg/cli/admin/upgrade/status/workerpool.go index f1a5a049ff..6b8632fa1e 100644 --- a/pkg/cli/admin/upgrade/status/workerpool.go +++ b/pkg/cli/admin/upgrade/status/workerpool.go @@ -596,7 +596,7 @@ Completion: {{ printf "%.0f" .Completion }}% Worker Status: {{ .NodesOverview.Total }} Total, {{ .NodesOverview.Available }} Available, {{ .NodesOverview.Progressing }} Progressing, {{ .NodesOverview.Outdated }} Outdated, {{ .NodesOverview.Draining }} Draining, {{ .NodesOverview.Excluded }} Excluded, {{ .NodesOverview.Degraded }} Degraded ` -func (pool *poolDisplayData) WriteNodes(w io.Writer) { +func (pool *poolDisplayData) WriteNodes(w io.Writer, detailed bool) { if pool.Name == mco.MachineConfigPoolMaster { fmt.Fprintf(w, "\nControl Plane Node") } else { @@ -610,7 +610,7 @@ func (pool *poolDisplayData) WriteNodes(w io.Writer) { _, _ = tabw.Write([]byte("\nNAME\tASSESSMENT\tPHASE\tVERSION\tEST\tMESSAGE\n")) var total, completed, available, progressing, outdated, draining, excluded int for i, node := range pool.Nodes { - if i >= 10 { + if i >= 10 && !detailed { // Limit displaying too many nodes // Display nodes in undesired states regardless their count if !node.isDegraded && (!node.isUnavailable || node.isUpdating) { @@ -645,6 +645,6 @@ func (pool *poolDisplayData) WriteNodes(w io.Writer) { } tabw.Flush() if total > 0 { - fmt.Fprintf(w, "...\nOmitted additional %d Total, %d Completed, %d Available, %d Progressing, %d Outdated, %d Draining, %d Excluded, and 0 Degraded nodes.\nPass along --details to see all information.\n", total, completed, available, progressing, outdated, draining, excluded) + fmt.Fprintf(w, "...\nOmitted additional %d Total, %d Completed, %d Available, %d Progressing, %d Outdated, %d Draining, %d Excluded, and 0 Degraded nodes.\nPass along --details=nodes to see all information.\n", total, completed, available, progressing, outdated, draining, excluded) } }