Skip to content

OCPBUGS-90086: [release-4.22] clear stale EtcdRecoveryActive failure condition when etcd is healthy #8806

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-bot merged 1 commit into from
Jul 27, 2026
Merged

OCPBUGS-90086: [release-4.22] clear stale EtcdRecoveryActive failure condition when etcd is healthy #8806

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 26 additions & 8 deletions hypershift-operator/controllers/hostedcluster/etcd_recovery.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,9 @@ func (r *HostedClusterReconciler) reconcileETCDMemberRecovery(ctx context.Contex

oldCondition := meta.FindStatusCondition(hcluster.Status.Conditions, string(hyperv1.EtcdRecoveryActive))

if oldCondition == nil || oldCondition.Status != etcdRecoveryActiveCondition.Status {
// The reason check is needed to transition from EtcdRecoveryJobFailed -> AsExpected
// when etcd self-heals (both use Status=False).
if oldCondition == nil || oldCondition.Status != etcdRecoveryActiveCondition.Status || oldCondition.Reason != etcdRecoveryActiveCondition.Reason {
meta.SetStatusCondition(&hcluster.Status.Conditions, etcdRecoveryActiveCondition)
if err := r.Client.Status().Update(ctx, hcluster); err != nil {
return nil, fmt.Errorf("failed to update etcd recovery job condition: %w", err)
Expand All @@ -87,7 +89,9 @@ func (r *HostedClusterReconciler) reconcileETCDMemberRecovery(ctx context.Contex

oldCondition := meta.FindStatusCondition(hcluster.Status.Conditions, string(hyperv1.EtcdRecoveryActive))

if oldCondition == nil || oldCondition.Status != etcdRecoveryActiveCondition.Status {
// The reason check is needed to transition from EtcdRecoveryJobFailed -> AsExpected
// when etcd self-heals (both use Status=False).
if oldCondition == nil || oldCondition.Status != etcdRecoveryActiveCondition.Status || oldCondition.Reason != etcdRecoveryActiveCondition.Reason {
meta.SetStatusCondition(&hcluster.Status.Conditions, etcdRecoveryActiveCondition)
if err := r.Client.Status().Update(ctx, hcluster); err != nil {
return nil, fmt.Errorf("failed to update etcd recovery job condition: %w", err)
Expand Down Expand Up @@ -132,12 +136,24 @@ func (r *HostedClusterReconciler) reconcileETCDMemberRecovery(ctx context.Contex
}

if failingEtcdPod == nil {
// No failing etcd pods detected
// However, if the statefulset is not reporting fully available, check later
if !fullyAvailable {
return &requeueAfter, nil
// Clear a stale EtcdRecoveryJobFailed condition when etcd is now healthy
// and no pods are failing — the cluster recovered on its own.
if fullyAvailable {
oldCondition := meta.FindStatusCondition(hcluster.Status.Conditions, string(hyperv1.EtcdRecoveryActive))
if oldCondition != nil && oldCondition.Reason == hyperv1.EtcdRecoveryJobFailedReason {
etcdRecoveryActiveCondition.Status = metav1.ConditionFalse
etcdRecoveryActiveCondition.Reason = hyperv1.AsExpectedReason
etcdRecoveryActiveCondition.Message = "Etcd cluster is healthy, clearing stale recovery failure condition."
etcdRecoveryActiveCondition.LastTransitionTime = r.now()
meta.SetStatusCondition(&hcluster.Status.Conditions, etcdRecoveryActiveCondition)
if err := r.Client.Status().Update(ctx, hcluster); err != nil {
return nil, fmt.Errorf("failed to update etcd recovery job condition: %w", err)
}
}
return nil, nil
}
return nil, nil
// Statefulset is not fully available yet, check later
return &requeueAfter, nil
}

log.Info("there are symptoms of etcd cluster degradation, triggering recovery job")
Expand Down Expand Up @@ -181,7 +197,9 @@ func (r *HostedClusterReconciler) reconcileETCDMemberRecovery(ctx context.Contex
// If the ETCD keeps failing and recovering, we can see the hcluster.Generation increasing indefinitely.
oldCondition := meta.FindStatusCondition(hcluster.Status.Conditions, string(hyperv1.EtcdRecoveryActive))

if oldCondition == nil || oldCondition.Status != etcdRecoveryActiveCondition.Status {
// The reason check is needed to transition from EtcdRecoveryJobFailed -> AsExpected
// when etcd self-heals (both use Status=False).
if oldCondition == nil || oldCondition.Status != etcdRecoveryActiveCondition.Status || oldCondition.Reason != etcdRecoveryActiveCondition.Reason {
meta.SetStatusCondition(&hcluster.Status.Conditions, etcdRecoveryActiveCondition)
if err := r.Client.Status().Update(ctx, hcluster); err != nil {
return nil, fmt.Errorf("failed to update etcd recovery job condition: %w", err)
Expand Down
Loading
Loading