diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output new file mode 100644 index 0000000000..35aa29b7c7 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output @@ -0,0 +1,41 @@ +An update is in progress for 1h58m50s: Unable to apply 4.14.1: wait has exceeded 40 minutes for these operators: etcd, kube-apiserver + +Failing=True: + + Reason: ClusterOperatorsDegraded + Message: Cluster operators etcd, kube-apiserver are degraded + + += Control Plane = +Assessment: Progressing +Completion: 97% +Duration: 1h58m50s +Operator Status: 33 Total, 32 Available, 1 Progressing, 4 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Pending +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? + += Update Health = +SINCE LEVEL IMPACT MESSAGE +58m18s Error API Availability Cluster Operator kube-apiserver is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) +58m18s Error API Availability Cluster Operator kube-controller-manager is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) +58m18s Error API Availability Cluster Operator kube-scheduler is degraded | NodeController_MasterNodesReady: NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) +58m38s Error API Availability Cluster Operator etcd is degraded | EtcdEndpoints_ErrorUpdatingEtcdEndpoints::EtcdMembers_UnhealthyMembers::NodeController_MasterNodesReady: EtcdEndpointsDegraded: EtcdEndpointsController can't evaluate whether quorum is safe: etcd cluster has quorum of 2 and 2 healthy members which is not fault tolerant: [{Member:ID:12895393557789359222 name:"ip-10-0-73-118.ec2.internal" peerURLs:"https://10.0.73.118:2380" clientURLs:"https://10.0.73.118:2379" Healthy:true Took:1.725492ms Error:} {Member:ID:13608765340770574953 name:"ip-10-0-0-60.ec2.internal" peerURLs:"https://10.0.0.60:2380" clientURLs:"https://10.0.0.60:2379" Healthy:true Took:1.542919ms Error:} {Member:ID:18044478200504924924 name:"ip-10-0-12-74.ec2.internal" peerURLs:"https://10.0.12.74:2380" clientURLs:"https://10.0.12.74:2379" Healthy:false Took: Error:create client failure: failed to make etcd client for endpoints [https://10.0.12.74:2379]: context deadline exceeded}] // EtcdMembersDegraded: 2 of 3 members are available, ip-10-0-12-74.ec2.internal is unhealthy // NodeControllerDegraded: The master nodes not ready: node "ip-10-0-12-74.ec2.internal" not ready since 2023-11-03 16:28:43 +0000 UTC because KubeletNotReady (container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?) +1h0m17s Error API Availability Cluster Operator control-plane-machine-set is unavailable | UnavailableReplicas: Missing 1 available replica(s) diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output new file mode 100644 index 0000000000..dd22bdd9e8 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output @@ -0,0 +1,31 @@ +An update is in progress for 6s: Working towards 4.14.1: 139 of 859 done (16% complete), waiting on kube-scheduler + += Control Plane = +Assessment: Progressing +Completion: 12% +Duration: 6s +Operator Status: 33 Total, 33 Available, 0 Progressing, 0 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0 ? + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Excluded +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 3 Excluded, 0 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Excluded Paused 4.14.0 - +ip-10-0-4-159.us-east-2.compute.internal Excluded Paused 4.14.0 - +ip-10-0-99-40.us-east-2.compute.internal Excluded Paused 4.14.0 - + += Update Health = +SINCE LEVEL IMPACT MESSAGE +- Warning Update Stalled Worker pool worker is paused | Outdated nodes in a paused pool will not be updated. diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output new file mode 100644 index 0000000000..b63a6ea757 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output @@ -0,0 +1,41 @@ +An update is in progress for 14m4s: Working towards 4.14.1: 734 of 859 done (85% complete), waiting on machine-config + += Control Plane = +Assessment: Progressing +Completion: 97% +Duration: 14m4s +Operator Status: 33 Total, 32 Available, 1 Progressing, 0 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-53-40.us-east-2.compute.internal Progressing Draining 4.14.0 +30m +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0 ? + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Progressing +Completion: 0% +Worker Status: 3 Total, 2 Available, 1 Progressing, 3 Outdated, 1 Draining, 0 Excluded, 0 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-4-159.us-east-2.compute.internal Progressing Draining 4.14.0 +30m +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.0 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0 ? + += Worker Pool = +Worker Pool: infra +Assessment: Progressing +Completion: 0% +Worker Status: 1 Total, 0 Available, 1 Progressing, 1 Outdated, 1 Draining, 0 Excluded, 0 Degraded + +Worker Pool Node +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-4-159-infra.us-east-2.compute.internal Progressing Draining 4.14.0 +30m + += Update Health = +SINCE LEVEL IMPACT MESSAGE +14m4s Info None Upgrade is proceeding well diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output new file mode 100644 index 0000000000..b52d104618 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output @@ -0,0 +1,31 @@ +An update is in progress for 52m56s: Working towards 4.15.0-ec.2: 357 of 1021 done (34% complete), waiting up to 40 minutes on cluster-api + += Control Plane = +Assessment: Progressing +Completion: 43% +Duration: 52m56s +Operator Status: 7 Total, 7 Available, 0 Progressing, 0 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Pending +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? +ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 ? + += Update Health = +SINCE LEVEL IMPACT MESSAGE +52m56s Info None Upgrade is proceeding well diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output new file mode 100644 index 0000000000..6567f43f52 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output @@ -0,0 +1,31 @@ +An update is in progress for 1m29s: Working towards 4.15.0-ec.2: 106 of 863 done (12% complete), waiting on etcd, kube-apiserver + += Control Plane = +Assessment: Progressing +Completion: 3% +Duration: 1m29s +Operator Status: 33 Total, 33 Available, 2 Progressing, 0 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.1 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.1 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.1 ? + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Pending +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.1 ? +ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.1 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.1 ? + += Update Health = +SINCE LEVEL IMPACT MESSAGE +1m29s Info None Upgrade is proceeding well diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output new file mode 100644 index 0000000000..af5e71c194 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output @@ -0,0 +1,31 @@ +An update is in progress for 58m53s: Working towards 4.15.0-ec.2: 110 of 863 done (12% complete), waiting up to 40 minutes on etcd + += Control Plane = +Assessment: Progressing +Completion: 97% +Duration: 58m53s +Operator Status: 33 Total, 31 Available, 1 Progressing, 1 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-53-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Pending +Completion: 0% +Worker Status: 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +ip-10-0-20-162.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-4-159.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? +ip-10-0-99-40.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? + += Update Health = +SINCE LEVEL IMPACT MESSAGE +20m24s Error API Availability Cluster Operator machine-config is unavailable | MachineConfigControllerFailed: Cluster not available for [{operator 4.14.0-rc.3}]: ControllerConfig.machineconfiguration.openshift.io "machine-config-controller" is invalid: [status.controllerCertificates[0].notAfter: Required value, status.controllerCertificates[0].notBefore: Required value, status.controllerCertificates[1].notAfter: Required value, status.controllerCertificates[1].notBefore: Required value, status.controllerCertificates[2].notAfter: Required value, status.controllerCertificates[2].notBefore: Required value, status.controllerCertificates[3].notAfter: Required value, status.controllerCertificates[3].notBefore: Required value, status.controllerCertificates[4].notAfter: Required value, status.controllerCertificates[4].notBefore: Required value, status.controllerCertificates[5].notAfter: Required value, status.controllerCertificates[5].notBefore: Required value, status.controllerCertificates[6].notAfter: Required value, status.controllerCertificates[6].notBefore: Required value, status.controllerCertificates[7].notAfter: Required value, status.controllerCertificates[7].notBefore: Required value, status.controllerCertificates[8].notAfter: Required value, status.controllerCertificates[8].notBefore: Required value, status.controllerCertificates[9].notAfter: Required value, status.controllerCertificates[9].notBefore: Required value, : Invalid value: "null": some validation rules were not checked because the object was invalid; correct the existing errors to complete validation] diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output new file mode 100644 index 0000000000..ffc2c2ee40 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output @@ -0,0 +1,100 @@ +An update is in progress for 4h3m46s: Error while reconciling 4.16.0-ec.3: the cluster operator machine-config is degraded + +Failing=True: + + Reason: ClusterOperatorDegraded + Message: Cluster operator machine-config is degraded + + += Control Plane = +Assessment: Completed +Completion: 100% +Duration: 3h30m31s +Operator Status: 36 Total, 36 Available, 0 Progressing, 1 Degraded + +Control Plane Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +build0-gstfj-m-0.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-m-1.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-m-2.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - + += Worker Upgrade = + += Worker Pool = +Worker Pool: worker +Assessment: Degraded +Completion: 39% +Worker Status: 59 Total, 46 Available, 5 Progressing, 36 Outdated, 12 Draining, 0 Excluded, 7 Degraded + +Worker Pool Nodes +NAME ASSESSMENT PHASE VERSION EST MESSAGE +build0-gstfj-ci-prowjobs-worker-b-9lztv Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-bg9f5 Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-prowjobs-worker-b-mrxwn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-4h7pn Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-jv5bg Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-b-kj6gk Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-c-dcz9p Degraded Draining 4.16.0-ec.2 ? failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information +build0-gstfj-ci-tests-worker-c-jq5rk Unavailable Updated 4.16.0-ec.3 - Node is unavailable +build0-gstfj-ci-tests-worker-c-2kz4m Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-tests-worker-c-55hpj Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-tests-worker-c-8tmjv Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-tests-worker-c-lfz8m Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-tests-worker-c-r55wv Progressing Draining 4.16.0-ec.2 +30m +build0-gstfj-ci-longtests-worker-c-vspm9 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-longtests-worker-d-hksc9 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-c-f6dkk Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-c-kkm72 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-c-p5df7 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-c-tgrsc Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-prowjobs-worker-d-ddnxd Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-2c4vg Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-5twk4 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-65n48 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-79nrd Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-82z85 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-glhn7 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-hnxl9 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-jpzfm Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-lxqgs Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-rqch4 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-c-zd9c7 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-6hddk Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-kc6xt Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-r2k42 Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-r9zqc Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-tests-worker-d-w2bgd Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-w-d-7rj9w.c.openshift-ci-build-farm.internal Outdated Pending 4.16.0-ec.2 ? +build0-gstfj-ci-builds-worker-b-65df4 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-b-dvrd2 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-b-f7rlh Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-b-m4vgr Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-b-mg8bd Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-c-2hzgm Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-c-dmjft Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-c-ng8s5 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-c-vmmfj Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-d-4pjlz Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-builds-worker-d-z49wm Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-tests-worker-b-d9vz2 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-tests-worker-b-fkn28 Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-tests-worker-b-mc7fm Completed Updated 4.16.0-ec.3 - +build0-gstfj-ci-tests-worker-b-pqz8t Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-f7dkq.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-infra-2stfh.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-infra-mhb8g.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-infra-n9kc5.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-b-mwf8t.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-c-nsmn5.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - +build0-gstfj-w-c-qjbdl.c.openshift-ci-build-farm.internal Completed Updated 4.16.0-ec.3 - + += Update Health = +SINCE LEVEL IMPACT MESSAGE +- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-9lztv is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-9lztv after 1 hour. Please see machine-config-controller logs for more information +- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-bg9f5 is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-bg9f5 after 1 hour. Please see machine-config-controller logs for more information +- Error Update Stalled Node build0-gstfj-ci-prowjobs-worker-b-mrxwn is degraded | failed to drain node: build0-gstfj-ci-prowjobs-worker-b-mrxwn after 1 hour. Please see machine-config-controller logs for more information +- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-4h7pn is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-4h7pn after 1 hour. Please see machine-config-controller logs for more information +- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-jv5bg is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-jv5bg after 1 hour. Please see machine-config-controller logs for more information +- Error Update Stalled Node build0-gstfj-ci-tests-worker-b-kj6gk is degraded | failed to drain node: build0-gstfj-ci-tests-worker-b-kj6gk after 1 hour. Please see machine-config-controller logs for more information +- Error Update Stalled Node build0-gstfj-ci-tests-worker-c-dcz9p is degraded | failed to drain node: build0-gstfj-ci-tests-worker-c-dcz9p after 1 hour. Please see machine-config-controller logs for more information +- Warning Update Speed Node build0-gstfj-ci-tests-worker-c-jq5rk is unavailable | Node is unavailable diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output index ebcd394dd6..7a200b483e 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output @@ -40,7 +40,7 @@ build0-gstfj-ci-tests-worker-c-2kz4m Progressing Draining 4.16.0-ec.2 build0-gstfj-ci-tests-worker-c-55hpj Progressing Draining 4.16.0-ec.2 +30m ... Omitted additional 49 Total, 22 Completed, 46 Available, 3 Progressing, 27 Outdated, 3 Draining, 0 Excluded, and 0 Degraded nodes. -Pass along --details to see all information. +Pass along --details=nodes to see all information. = Update Health = SINCE LEVEL IMPACT MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/README.md b/pkg/cli/admin/upgrade/status/examples/README.md index 87857d66c4..4c9fd2f30e 100644 --- a/pkg/cli/admin/upgrade/status/examples/README.md +++ b/pkg/cli/admin/upgrade/status/examples/README.md @@ -1,12 +1,13 @@ # Examples for `oc adm upgrade status` -Each example consists of five inputs and one output, matched by a common substring: +Each example consists of five inputs and two outputs, matched by a common substring: 1. `TESTCASE-cv.yaml`(input): ClusterVersion object (created by `oc get clusterversion version -o yaml`) 2. `TESTCASE-co.yaml`(input): list of ClusterOperators (created by `oc get clusteroperators -o yaml`) 3. `TESTCASE-mc.yaml`(input): list of MachineConfigs (created by `oc get oc get machineconfigs -o yaml`) 4. `TESTCASE-mcp.yaml`(input): list of MachineConfigPools (created by `oc get machineconfigpools -o yaml`) 5. `TESTCASE-node.yaml`(input): list of Nodes (created by `oc get nodes -o yaml`) -6. `TESTCASE.out`(output): expected output of `oc adm upgrade status` for the two outputs +6. `TESTCASE.output`(output): expected output of `oc adm upgrade status` +7. `TESTCASE.detailed-output`(output): expected output of `oc adm upgrade status --detailed=all` The `TestExamples` test in `examples_test.go` file above validates all examples. When the testcase is executed with a non-empty `UPDATE` environmental variable, it will update the `TESTCASE.out` diff --git a/pkg/cli/admin/upgrade/status/examples/not-upgrading.detailed-output b/pkg/cli/admin/upgrade/status/examples/not-upgrading.detailed-output new file mode 100644 index 0000000000..add8b8cab0 --- /dev/null +++ b/pkg/cli/admin/upgrade/status/examples/not-upgrading.detailed-output @@ -0,0 +1,4 @@ +The cluster version is not updating (Progressing=False). + + Reason: + Message: Cluster version is 4.14.1 diff --git a/pkg/cli/admin/upgrade/status/examples_test.go b/pkg/cli/admin/upgrade/status/examples_test.go index 7782c0d8cb..1ba2cce7bf 100644 --- a/pkg/cli/admin/upgrade/status/examples_test.go +++ b/pkg/cli/admin/upgrade/status/examples_test.go @@ -3,6 +3,7 @@ package status import ( "bytes" "context" + "fmt" "os" "path/filepath" "strings" @@ -11,9 +12,9 @@ import ( "github.com/google/go-cmp/cmp" ) -func compareWithFixture(t *testing.T, actualOut []byte, cvPath string) { +func compareWithFixture(t *testing.T, actualOut []byte, cvPath string, outputSuffix string) { t.Helper() - expectedOutPath := strings.Replace(cvPath, "-cv.yaml", ".output", 1) + expectedOutPath := strings.Replace(cvPath, "-cv.yaml", outputSuffix, 1) if update := os.Getenv("UPDATE"); update != "" { if err := os.WriteFile(expectedOutPath, actualOut, 0644); err != nil { @@ -24,10 +25,15 @@ func compareWithFixture(t *testing.T, actualOut []byte, cvPath string) { expectedOut, err := os.ReadFile(expectedOutPath) if err != nil { - t.Fatalf("Error when reading output fixture: %v", err) + if !os.IsNotExist(err) { + t.Fatalf("Error when reading output fixture: %v", err) + } else { + t.Fatalf("Output file %s does not exist. You may rerun this test with UPDATE=true to create output file with the following actual output:\n%s", expectedOutPath, actualOut) + } } + if diff := cmp.Diff(string(expectedOut), string(actualOut)); diff != "" { - t.Errorf("Output differs from expected:\n%s", diff) + t.Errorf("Output differs from expected (%s):\n%s", filepath.Base(expectedOutPath), diff) } } @@ -37,25 +43,47 @@ func TestExamples(t *testing.T) { t.Fatalf("Error when listing examples: %v", err) } + variants := []struct { + name string + detailed string + outputSuffix string + }{ + { + name: "normal output", + detailed: "none", + outputSuffix: ".output", + }, + { + name: "detailed output", + detailed: "all", + outputSuffix: ".detailed-output", + }, + } + for _, cv := range cvs { cv := cv - t.Run(cv, func(t *testing.T) { - t.Parallel() - - opts := &options{mockData: mockData{cvPath: cv}} - if err := opts.Complete(nil, nil, nil); err != nil { - t.Fatalf("Error when completing options: %v", err) - } + for _, variant := range variants { + variant := variant + t.Run(fmt.Sprintf("%s-%s", cv, variant.name), func(t *testing.T) { + t.Parallel() + opts := &options{ + mockData: mockData{cvPath: cv}, + detailedOutput: variant.detailed, + } + if err := opts.Complete(nil, nil, nil); err != nil { + t.Fatalf("Error when completing options: %v", err) + } - var stdout, stderr bytes.Buffer - opts.Out = &stdout - opts.ErrOut = &stderr + var stdout, stderr bytes.Buffer + opts.Out = &stdout + opts.ErrOut = &stderr - if err := opts.Run(context.Background()); err != nil { - t.Fatalf("Error when running: %v", err) - } + if err := opts.Run(context.Background()); err != nil { + t.Fatalf("Error when running: %v", err) + } - compareWithFixture(t, stdout.Bytes(), cv) - }) + compareWithFixture(t, stdout.Bytes(), cv, variant.outputSuffix) + }) + } } } diff --git a/pkg/cli/admin/upgrade/status/status.go b/pkg/cli/admin/upgrade/status/status.go index c50e533f79..0da800b3e9 100644 --- a/pkg/cli/admin/upgrade/status/status.go +++ b/pkg/cli/admin/upgrade/status/status.go @@ -11,6 +11,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/cli-runtime/pkg/genericiooptions" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" kcmdutil "k8s.io/kubectl/pkg/cmd/util" @@ -34,6 +35,15 @@ func newOptions(streams genericiooptions.IOStreams) *options { } } +const ( + detailedOutputNone = "none" + detailedOutputAll = "all" + detailedOutputNodes = "nodes" + detailedOutputHealth = "health" +) + +var detailedOutputAllValues = []string{detailedOutputNone, detailedOutputAll, detailedOutputNodes, detailedOutputHealth} + func New(f kcmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Command { o := newOptions(streams) cmd := &cobra.Command{ @@ -49,6 +59,7 @@ func New(f kcmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Command // TODO: We can remove these flags once the idea about `oc adm upgrade status` stabilizes and the command // is promoted out of the OC_ENABLE_CMD_UPGRADE_STATUS feature gate flags.StringVar(&o.mockData.cvPath, "mock-clusterversion", "", "Path to a YAML ClusterVersion object to use for testing (will be removed later). Files in the same directory with the same name and suffixes -co.yaml, -mcp.yaml, -mc.yaml, and -node.yaml are required.") + flags.StringVar(&o.detailedOutput, "detailed", "none", fmt.Sprintf("Show detailed output in selected section. One of: %s", strings.Join(detailedOutputAllValues, ", "))) return cmd } @@ -56,18 +67,27 @@ func New(f kcmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Command type options struct { genericiooptions.IOStreams - mockData mockData + mockData mockData + detailedOutput string ConfigClient configv1client.Interface CoreClient corev1client.CoreV1Interface MachineConfigClient machineconfigv1client.Interface } +func (o *options) enabledDetailed(what string) bool { + return o.detailedOutput == detailedOutputAll || o.detailedOutput == what +} + func (o *options) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if len(args) > 0 { return kcmdutil.UsageErrorf(cmd, "positional arguments given") } + if !sets.New[string](detailedOutputAllValues...).Has(o.detailedOutput) { + return fmt.Errorf("invalid value for --detailed: %s (must be one of %s)", o.detailedOutput, strings.Join(detailedOutputAllValues, ", ")) + } + cvSuffix := "-cv.yaml" if o.mockData.cvPath != "" { o.mockData.operatorsPath = strings.Replace(o.mockData.cvPath, cvSuffix, "-co.yaml", 1) @@ -244,13 +264,13 @@ func (o *options) Run(ctx context.Context) error { updateInsights = append(updateInsights, insights...) fmt.Fprintf(o.Out, "\n") _ = controlPlaneStatusData.Write(o.Out) - controlPlanePoolStatusData.WriteNodes(o.Out) + controlPlanePoolStatusData.WriteNodes(o.Out, o.enabledDetailed(detailedOutputNodes)) fmt.Fprintf(o.Out, "\n= Worker Upgrade =\n") for _, pool := range workerPoolsStatusData { fmt.Fprintf(o.Out, "\n") _ = pool.WritePool(o.Out) - pool.WriteNodes(o.Out) + pool.WriteNodes(o.Out, o.enabledDetailed(detailedOutputNodes)) } fmt.Fprintf(o.Out, "\n") diff --git a/pkg/cli/admin/upgrade/status/workerpool.go b/pkg/cli/admin/upgrade/status/workerpool.go index d61d26efad..1fc8d54aba 100644 --- a/pkg/cli/admin/upgrade/status/workerpool.go +++ b/pkg/cli/admin/upgrade/status/workerpool.go @@ -453,7 +453,7 @@ Completion: {{ printf "%.0f" .Completion }}% Worker Status: {{ .NodesOverview.Total }} Total, {{ .NodesOverview.Available }} Available, {{ .NodesOverview.Progressing }} Progressing, {{ .NodesOverview.Outdated }} Outdated, {{ .NodesOverview.Draining }} Draining, {{ .NodesOverview.Excluded }} Excluded, {{ .NodesOverview.Degraded }} Degraded ` -func (pool *poolDisplayData) WriteNodes(w io.Writer) { +func (pool *poolDisplayData) WriteNodes(w io.Writer, detailed bool) { if pool.Name == mco.MachineConfigPoolMaster { fmt.Fprintf(w, "\nControl Plane Node") } else { @@ -467,8 +467,8 @@ func (pool *poolDisplayData) WriteNodes(w io.Writer) { _, _ = tabw.Write([]byte("\nNAME\tASSESSMENT\tPHASE\tVERSION\tEST\tMESSAGE\n")) var total, completed, available, progressing, outdated, draining, excluded int for i, node := range pool.Nodes { - if i >= 10 { - // Limit displaying too many nodes + if !detailed && i >= 10 { + // Limit displaying too many nodes when not in detailed mode // Display nodes in undesired states regardless their count if !node.isDegraded && (!node.isUnavailable || node.isUpdating) { total++ @@ -502,6 +502,6 @@ func (pool *poolDisplayData) WriteNodes(w io.Writer) { } tabw.Flush() if total > 0 { - fmt.Fprintf(w, "...\nOmitted additional %d Total, %d Completed, %d Available, %d Progressing, %d Outdated, %d Draining, %d Excluded, and 0 Degraded nodes.\nPass along --details to see all information.\n", total, completed, available, progressing, outdated, draining, excluded) + fmt.Fprintf(w, "...\nOmitted additional %d Total, %d Completed, %d Available, %d Progressing, %d Outdated, %d Draining, %d Excluded, and 0 Degraded nodes.\nPass along --details=nodes to see all information.\n", total, completed, available, progressing, outdated, draining, excluded) } }