Skip to content

Fix snapshot reconcile retry #9318

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brandond merged 2 commits into from
Feb 7, 2024
Merged

Fix snapshot reconcile retry #9318

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f601dd8
Fix excessive retry on snapshot reconcile
brandond Jan 30, 2024
765d7b0
Retry startup snapshot reconcile
brandond Jan 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions pkg/cluster/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"net/url"
"runtime"
"strings"
"time"

"github.com/k3s-io/k3s/pkg/clientaccess"
"github.com/k3s-io/k3s/pkg/cluster/managed"
Expand All @@ -13,6 +14,7 @@ import (
"github.com/k3s-io/kine/pkg/endpoint"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/util/wait"
utilsnet "k8s.io/utils/net"
)

Expand Down Expand Up @@ -107,11 +109,14 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
}

if !c.config.EtcdDisableSnapshots {
if err := c.managedDB.ReconcileSnapshotData(ctx); err != nil {
logrus.Errorf("Failed to record snapshots for cluster: %v", err)
}
wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (bool, error) {
err := c.managedDB.ReconcileSnapshotData(ctx)
if err != nil {
logrus.Errorf("Failed to record snapshots for cluster: %v", err)
}
return err == nil, nil
})
}

return
default:
runtime.Gosched()
Expand Down
9 changes: 8 additions & 1 deletion pkg/etcd/snapshot_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,13 @@ const (
var (
snapshotConfigMapName = version.Program + "-etcd-snapshots"
errNotReconciled = errors.New("no nodes have reconciled ETCDSnapshotFile resources")
reconcileBackoff = wait.Backoff{
Steps: 9,
Duration: 10 * time.Millisecond,
Factor: 3.0,
Jitter: 0.1,
Cap: 30 * time.Second,
}
)

type etcdSnapshotHandler struct {
Expand Down Expand Up @@ -62,7 +69,7 @@ func (e *etcdSnapshotHandler) sync(key string, esf *apisv1.ETCDSnapshotFile) (*a
err := e.reconcile()
if err == errNotReconciled {
logrus.Debugf("Failed to reconcile snapshot ConfigMap: %v, requeuing", err)
e.snapshots.Enqueue(key)
e.snapshots.EnqueueAfter(key, reconcileBackoff.Step())
return nil, nil
}
return nil, err
Expand Down