Skip to content

timeout for quorum check #942

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tjungblu wants to merge 1 commit into from
Closed

timeout for quorum check #942

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion pkg/operator/ceohelpers/bootstrap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package ceohelpers
import (
"context"
"fmt"
"time"

configv1listers "github.com/openshift/client-go/config/listers/config/v1"
"github.com/openshift/library-go/pkg/operator/v1helpers"
Expand Down Expand Up @@ -101,6 +102,7 @@ func GetBootstrapScalingStrategy(staticPodClient v1helpers.StaticPodOperatorClie
// the etcd cluster based on the scaling strategy in use, and otherwise will return
// an error explaining why it's unsafe to scale.
func CheckSafeToScaleCluster(
ctx context.Context,
configmapLister corev1listers.ConfigMapLister,
staticPodClient v1helpers.StaticPodOperatorClient,
namespaceLister corev1listers.NamespaceLister,
Expand Down Expand Up @@ -146,7 +148,10 @@ func CheckSafeToScaleCluster(
return fmt.Errorf("CheckSafeToScaleCluster %d nodes are required, but only %d are available", minimumNodes, nodeCount)
}

memberHealth, err := etcdClient.MemberHealth(context.Background())
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()

memberHealth, err := etcdClient.MemberHealth(ctx)
if err != nil {
return fmt.Errorf("CheckSafeToScaleCluster couldn't determine member health: %w", err)
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/operator/ceohelpers/bootstrap_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package ceohelpers

import (
"context"
"fmt"
"testing"

Expand Down Expand Up @@ -369,6 +370,7 @@ func Test_CheckSafeToScaleCluster(t *testing.T) {
}

actualErr := CheckSafeToScaleCluster(
context.Background(),
fakeConfigMapLister,
fakeStaticPodClient,
fakeNamespaceMapLister,
Expand Down
8 changes: 5 additions & 3 deletions pkg/operator/ceohelpers/qourum_check.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package ceohelpers

import (
"context"

configv1listers "github.com/openshift/client-go/config/listers/config/v1"
"github.com/openshift/library-go/pkg/operator/v1helpers"
corev1listers "k8s.io/client-go/listers/core/v1"
Expand All @@ -12,7 +14,7 @@ type QuorumChecker interface {
// IsSafeToUpdateRevision checks the current etcd cluster and returns true if the cluster can tolerate the
// loss of a single etcd member. Such loss is common during new static pod revision.
// Returns True when it is absolutely safe, false if not. Error otherwise, which always indicates it is unsafe.
IsSafeToUpdateRevision() (bool, error)
IsSafeToUpdateRevision(ctx context.Context) (bool, error)
}

// QuorumCheck is just a convenience struct around bootstrap.go
Expand All @@ -24,8 +26,8 @@ type QuorumCheck struct {
etcdClient etcdcli.EtcdClient
}

func (c *QuorumCheck) IsSafeToUpdateRevision() (bool, error) {
err := CheckSafeToScaleCluster(c.configMapLister, c.operatorClient, c.namespaceLister, c.infraLister, c.etcdClient)
func (c *QuorumCheck) IsSafeToUpdateRevision(ctx context.Context) (bool, error) {
err := CheckSafeToScaleCluster(ctx, c.configMapLister, c.operatorClient, c.namespaceLister, c.infraLister, c.etcdClient)
if err != nil {
return false, err
}
Expand Down
3 changes: 2 additions & 1 deletion pkg/operator/ceohelpers/qourum_check_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package ceohelpers

import (
"context"
"fmt"
"testing"

Expand Down Expand Up @@ -137,7 +138,7 @@ func TestQuorumCheck_IsSafeToUpdateRevision(t *testing.T) {
fakeOperatorClient,
fakeEtcdClient)

safe, err := quorumChecker.IsSafeToUpdateRevision()
safe, err := quorumChecker.IsSafeToUpdateRevision(context.Background())
assert.Equal(t, scenario.expectedErr, err)
assert.Equal(t, scenario.safe, safe)
})
Expand Down
2 changes: 1 addition & 1 deletion pkg/operator/etcdcertsigner/etcdcertsignercontroller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func NewEtcdCertSignerController(
}

func (c *EtcdCertSignerController) sync(ctx context.Context, syncCtx factory.SyncContext) error {
safe, err := c.quorumChecker.IsSafeToUpdateRevision()
safe, err := c.quorumChecker.IsSafeToUpdateRevision(ctx)
if err != nil {
return fmt.Errorf("EtcdCertSignerController can't evaluate whether quorum is safe: %w", err)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/operator/etcdendpointscontroller/etcdendpointscontroller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ func (c *EtcdEndpointsController) syncConfigMap(ctx context.Context, recorder ev

required.Data = endpointAddresses

safe, err := c.quorumChecker.IsSafeToUpdateRevision()
safe, err := c.quorumChecker.IsSafeToUpdateRevision(ctx)
if err != nil {
return fmt.Errorf("EtcdEndpointsController can't evaluate whether quorum is safe: %w", err)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/operator/targetconfigcontroller/targetconfigcontroller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func NewTargetConfigController(
}

func (c TargetConfigController) sync(ctx context.Context, syncCtx factory.SyncContext) error {
safe, err := c.quorumChecker.IsSafeToUpdateRevision()
safe, err := c.quorumChecker.IsSafeToUpdateRevision(ctx)
if err != nil {
return fmt.Errorf("TargetConfigController can't evaluate whether quorum is safe: %w", err)
}
Expand Down