From 2cbd9eee1713ca18ebb8ae12703aed4093943874 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 30 Jan 2024 23:33:39 +0000 Subject: [PATCH 01/10] Fix excessive retry on snapshot reconcile Signed-off-by: Brad Davidson (cherry picked from commit 4005600d4e3ebf429632504f21deb893ae28893c) Signed-off-by: Brad Davidson --- pkg/etcd/snapshot_controller.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/etcd/snapshot_controller.go b/pkg/etcd/snapshot_controller.go index 295f0117e43e..d4b22755e03f 100644 --- a/pkg/etcd/snapshot_controller.go +++ b/pkg/etcd/snapshot_controller.go @@ -33,6 +33,13 @@ const ( var ( snapshotConfigMapName = version.Program + "-etcd-snapshots" errNotReconciled = errors.New("no nodes have reconciled ETCDSnapshotFile resources") + reconcileBackoff = wait.Backoff{ + Steps: 9, + Duration: 10 * time.Millisecond, + Factor: 3.0, + Jitter: 0.1, + Cap: 30 * time.Second, + } ) type etcdSnapshotHandler struct { @@ -62,7 +69,7 @@ func (e *etcdSnapshotHandler) sync(key string, esf *apisv1.ETCDSnapshotFile) (*a err := e.reconcile() if err == errNotReconciled { logrus.Debugf("Failed to reconcile snapshot ConfigMap: %v, requeuing", err) - e.snapshots.Enqueue(key) + e.snapshots.EnqueueAfter(key, reconcileBackoff.Step()) return nil, nil } return nil, err From f5cbabe96a67fcc5d230336012b72758cd503e44 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 30 Jan 2024 23:47:18 +0000 Subject: [PATCH 02/10] Retry startup snapshot reconcile The reconcile may run before the kubelet has created the node object; retry until it succeeds Signed-off-by: Brad Davidson (cherry picked from commit 82e3c32c9f837065acce2d86e01ba47447f14cb8) Signed-off-by: Brad Davidson --- pkg/cluster/cluster.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 7e3ecf268695..d87c2d9fa617 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -5,6 +5,7 @@ import ( "net/url" "runtime" "strings" + "time" "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/cluster/managed" @@ -13,6 +14,7 @@ import ( "github.com/k3s-io/kine/pkg/endpoint" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/wait" utilsnet "k8s.io/utils/net" ) @@ -107,11 +109,14 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) { } if !c.config.EtcdDisableSnapshots { - if err := c.managedDB.ReconcileSnapshotData(ctx); err != nil { - logrus.Errorf("Failed to record snapshots for cluster: %v", err) - } + wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (bool, error) { + err := c.managedDB.ReconcileSnapshotData(ctx) + if err != nil { + logrus.Errorf("Failed to record snapshots for cluster: %v", err) + } + return err == nil, nil + }) } - return default: runtime.Gosched() From 3a82b55392a73dc8520404ea08ab45c316910e7a Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 30 Jan 2024 22:44:58 +0000 Subject: [PATCH 03/10] Add check for etcd-snapshot-dir and fix panic in Walk Signed-off-by: Brad Davidson (cherry picked from commit 6ec1926f884a74e069885688e1f7595f50e4621f) Signed-off-by: Brad Davidson --- pkg/etcd/snapshot.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/etcd/snapshot.go b/pkg/etcd/snapshot.go index afdca6e2960d..d9f5f7689c1e 100644 --- a/pkg/etcd/snapshot.go +++ b/pkg/etcd/snapshot.go @@ -245,7 +245,13 @@ func (e *ETCD) Snapshot(ctx context.Context) error { snapshotDir, err := snapshotDir(e.config, true) if err != nil { - return errors.Wrap(err, "failed to get the snapshot dir") + return errors.Wrap(err, "failed to get etcd-snapshot-dir") + } + + if info, err := os.Stat(snapshotDir); err != nil { + return errors.Wrapf(err, "failed to stat etcd-snapshot-dir %s", snapshotDir) + } else if !info.IsDir() { + return fmt.Errorf("etcd-snapshot-dir %s is not a directory", snapshotDir) } cfg, err := getClientConfig(ctx, e.config) @@ -436,7 +442,7 @@ func (e *ETCD) listLocalSnapshots() (map[string]snapshotFile, error) { } if err := filepath.Walk(snapshotDir, func(path string, file os.FileInfo, err error) error { - if file.IsDir() || err != nil { + if err != nil || file.IsDir() { return err } From 92e582815ddf89668143b4f5cc5be58d5d91bffc Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 16 Jan 2024 21:43:59 +0000 Subject: [PATCH 04/10] Bump CNI plugins to v1.4.0 Ref: https://github.com/rancher/plugins/compare/v1.3.0-k3s1...v1.4.0-k3s2 Signed-off-by: Brad Davidson (cherry picked from commit 77ba9904d1b6e3b96d65ed2d7eeb761d08590a97) Signed-off-by: Brad Davidson --- scripts/version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/version.sh b/scripts/version.sh index f7bc93fef808..a67f0d560794 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -68,7 +68,7 @@ if [ -z "$VERSION_CRI_DOCKERD" ]; then VERSION_CRI_DOCKERD="v0.0.0" fi -VERSION_CNIPLUGINS="v1.3.0-k3s1" +VERSION_CNIPLUGINS="v1.4.0-k3s2" VERSION_KUBE_ROUTER=$(get-module-version github.com/cloudnativelabs/kube-router/v2) if [ -z "$VERSION_KUBE_ROUTER" ]; then From bacc86efb80a771705785eb426fbcba5e0ab3d0e Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Mon, 5 Feb 2024 20:25:08 +0000 Subject: [PATCH 05/10] Fix issue with coredns node hosts controller The nodes controller was reading from the configmaps cache, but doesn't add any handlers, so if no other controller added configmap handlers, the cache would remain empty. Signed-off-by: Brad Davidson (cherry picked from commit 888f866dae243dcc205e31b6b1d8716bd00af4dc) Signed-off-by: Brad Davidson --- pkg/node/controller.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/node/controller.go b/pkg/node/controller.go index ef111541c900..c52203279838 100644 --- a/pkg/node/controller.go +++ b/pkg/node/controller.go @@ -9,6 +9,7 @@ import ( coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" "github.com/sirupsen/logrus" core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func Register(ctx context.Context, @@ -76,13 +77,12 @@ func (h *handler) updateCoreDNSConfigMap(nodeName, nodeAddress string, removed b return nil } - configMapCache, err := h.configMaps.Cache().Get("kube-system", "coredns") - if err != nil || configMapCache == nil { + configMap, err := h.configMaps.Get("kube-system", "coredns", metav1.GetOptions{}) + if err != nil || configMap == nil { logrus.Warn(errors.Wrap(err, "Unable to fetch coredns config map")) return nil } - configMap := configMapCache.DeepCopy() hosts := configMap.Data["NodeHosts"] hostsMap := map[string]string{} @@ -116,6 +116,10 @@ func (h *handler) updateCoreDNSConfigMap(nodeName, nodeAddress string, removed b for host, ip := range hostsMap { newHosts += ip + " " + host + "\n" } + + if configMap.Data == nil { + configMap.Data = map[string]string{} + } configMap.Data["NodeHosts"] = newHosts if _, err := h.configMaps.Update(configMap); err != nil { From 12439f565f42fc7328bb19691ddbe1f0b01be821 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 16 Jan 2024 22:43:08 +0000 Subject: [PATCH 06/10] Fix ipv6 endpoint address selection for on-demand snapshots Signed-off-by: Brad Davidson (cherry picked from commit 8224a3a7f6deb0ea75636d301396031e0a298456) Signed-off-by: Brad Davidson --- pkg/cli/etcdsnapshot/etcd_snapshot.go | 45 ++++++++++++++++++++++++++- pkg/cli/server/server.go | 1 + pkg/etcd/etcd.go | 11 ++++--- pkg/etcd/etcd_test.go | 2 +- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/pkg/cli/etcdsnapshot/etcd_snapshot.go b/pkg/cli/etcdsnapshot/etcd_snapshot.go index 1f83b98c5a22..495e471a2e90 100644 --- a/pkg/cli/etcdsnapshot/etcd_snapshot.go +++ b/pkg/cli/etcdsnapshot/etcd_snapshot.go @@ -3,8 +3,8 @@ package etcdsnapshot import ( "context" "encoding/json" - "errors" "fmt" + "net" "os" "path/filepath" "sort" @@ -17,7 +17,9 @@ import ( daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/etcd" "github.com/k3s-io/k3s/pkg/server" + "github.com/k3s-io/k3s/pkg/util" util2 "github.com/k3s-io/k3s/pkg/util" + "github.com/pkg/errors" "github.com/rancher/wrangler/pkg/signals" "github.com/urfave/cli" "gopkg.in/yaml.v2" @@ -52,6 +54,7 @@ func commandSetup(app *cli.Context, cfg *cmds.Server, config *server.Config) (*e config.DisableAgent = true config.ControlConfig.DataDir = dataDir + config.ControlConfig.BindAddress = cfg.BindAddress config.ControlConfig.EtcdSnapshotName = cfg.EtcdSnapshotName config.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir config.ControlConfig.EtcdSnapshotCompress = cfg.EtcdSnapshotCompress @@ -73,6 +76,46 @@ func commandSetup(app *cli.Context, cfg *cmds.Server, config *server.Config) (*e config.ControlConfig.Runtime.ClientETCDKey = filepath.Join(dataDir, "tls", "etcd", "client.key") config.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig") + // We need to go through defaulting of cluster addresses to ensure that the etcd config for the standalone + // command uses the same endpoint selection logic as it does when starting up the full server. Specifically, + // we need to set an IPv6 service CIDR on IPv6-only or IPv6-first nodes, as the etcd default endpoints check + // the service CIDR primary addresss family to determine what loopback address to use. + _, nodeIPs, err := util.GetHostnameAndIPs(cmds.AgentConfig.NodeName, cmds.AgentConfig.NodeIP) + if err != nil { + return nil, err + } + + // configure ClusterIPRanges. Use default 10.42.0.0/16 or fd00:42::/56 if user did not set it + _, defaultClusterCIDR, defaultServiceCIDR, _ := util.GetDefaultAddresses(nodeIPs[0]) + if len(cfg.ClusterCIDR) == 0 { + cfg.ClusterCIDR.Set(defaultClusterCIDR) + } + for _, cidr := range util.SplitStringSlice(cfg.ClusterCIDR) { + _, parsed, err := net.ParseCIDR(cidr) + if err != nil { + return nil, errors.Wrapf(err, "invalid cluster-cidr %s", cidr) + } + config.ControlConfig.ClusterIPRanges = append(config.ControlConfig.ClusterIPRanges, parsed) + } + + // set ClusterIPRange to the first address (first defined IPFamily is preferred) + config.ControlConfig.ClusterIPRange = config.ControlConfig.ClusterIPRanges[0] + + // configure ServiceIPRanges. Use default 10.43.0.0/16 or fd00:43::/112 if user did not set it + if len(cfg.ServiceCIDR) == 0 { + cfg.ServiceCIDR.Set(defaultServiceCIDR) + } + for _, cidr := range util.SplitStringSlice(cfg.ServiceCIDR) { + _, parsed, err := net.ParseCIDR(cidr) + if err != nil { + return nil, errors.Wrapf(err, "invalid service-cidr %s", cidr) + } + config.ControlConfig.ServiceIPRanges = append(config.ControlConfig.ServiceIPRanges, parsed) + } + + // set ServiceIPRange to the first address (first defined IPFamily is preferred) + config.ControlConfig.ServiceIPRange = config.ControlConfig.ServiceIPRanges[0] + e := etcd.NewETCD() if err := e.SetControlConfig(&config.ControlConfig); err != nil { return nil, err diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 1bb718625b57..25f48aaa8c42 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -441,6 +441,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.DisableControllerManager = true serverConfig.ControlConfig.DisableScheduler = true serverConfig.ControlConfig.DisableCCM = true + serverConfig.ControlConfig.DisableServiceLB = true // If the supervisor and apiserver are on the same port, everything is running embedded // and we don't need the kubelet or containerd up to perform a cluster reset. diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 87541284a10e..9a2e5fb01035 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -400,6 +400,7 @@ func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error { if err := os.WriteFile(e.ResetFile(), []byte{}, 0600); err != nil { return err } + return e.newCluster(ctx, true) } @@ -757,7 +758,7 @@ func getAdvertiseAddress(advertiseIP string) (string, error) { // newCluster returns options to set up etcd for a new cluster func (e *ETCD) newCluster(ctx context.Context, reset bool) error { - logrus.Infof("Starting etcd for new cluster") + logrus.Infof("Starting etcd for new cluster, cluster-reset=%v", reset) err := e.cluster(ctx, reset, executor.InitialOptions{ AdvertisePeerURL: e.peerURL(), Cluster: fmt.Sprintf("%s=%s", e.name, e.peerURL()), @@ -766,8 +767,10 @@ func (e *ETCD) newCluster(ctx context.Context, reset bool) error { if err != nil { return err } - if err := e.migrateFromSQLite(ctx); err != nil { - return fmt.Errorf("failed to migrate content from sqlite to etcd: %w", err) + if !reset { + if err := e.migrateFromSQLite(ctx); err != nil { + return fmt.Errorf("failed to migrate content from sqlite to etcd: %w", err) + } } return nil } @@ -848,7 +851,7 @@ func (e *ETCD) clientURL() string { // on other nodes connect mid-process. func (e *ETCD) advertiseClientURLs(reset bool) string { if reset { - return fmt.Sprintf("https://%s", net.JoinHostPort(e.config.Loopback(true), "2379")) + return fmt.Sprintf("https://%s:2379", e.config.Loopback(true)) } return e.clientURL() } diff --git a/pkg/etcd/etcd_test.go b/pkg/etcd/etcd_test.go index c3af06284a3d..7293d99c535f 100644 --- a/pkg/etcd/etcd_test.go +++ b/pkg/etcd/etcd_test.go @@ -47,7 +47,7 @@ func generateTestConfig() *config.Control { EtcdSnapshotRetention: 5, EtcdS3Endpoint: "s3.amazonaws.com", EtcdS3Region: "us-east-1", - SANs: []string{"127.0.0.1"}, + SANs: []string{"127.0.0.1", mustGetAddress()}, CriticalControlArgs: criticalControlArgs, } } From 6425ae4782da9f02b190f3ab1a82e7c152aac655 Mon Sep 17 00:00:00 2001 From: Oleg Matskiv Date: Mon, 29 Jan 2024 21:26:25 +0100 Subject: [PATCH 07/10] Don't verify the node password if the local host is not running an agent Signed-off-by: Oleg Matskiv (cherry picked from commit e3b237fc35d9eee0d5912533d1035d71163baf9d) Signed-off-by: Brad Davidson --- pkg/server/router.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/server/router.go b/pkg/server/router.go index d1e112ecec6b..e336e3ca0556 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -488,6 +488,11 @@ func passwordBootstrap(ctx context.Context, config *Config) nodePassBootstrapper } func verifyLocalPassword(ctx context.Context, config *Config, mu *sync.Mutex, deferredNodes map[string]bool, node *nodeInfo) (string, int, error) { + // do not attempt to verify the node password if the local host is not running an agent and does not have a node resource. + if config.DisableAgent { + return node.Name, http.StatusOK, nil + } + // use same password file location that the agent creates nodePasswordRoot := "/" if config.ControlConfig.Rootless { From d07fe5815850bc97dcd1fd8abc9892c8831a225b Mon Sep 17 00:00:00 2001 From: Edgar Lee Date: Fri, 26 Jan 2024 22:49:37 -0500 Subject: [PATCH 08/10] Expose rootless state dir under ~/.rancher/k3s/rootless Signed-off-by: Edgar Lee (cherry picked from commit 14c6c63b308f11d4917bfc6249656174caeb64be) Signed-off-by: Brad Davidson --- pkg/rootless/rootless.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/rootless/rootless.go b/pkg/rootless/rootless.go index 97266d7eafa6..f584e7b4a7cd 100644 --- a/pkg/rootless/rootless.go +++ b/pkg/rootless/rootless.go @@ -132,11 +132,6 @@ func createParentOpt(driver portDriver, stateDir string, enableIPv6 bool) (*pare return nil, errors.Wrapf(err, "failed to mkdir %s", stateDir) } - stateDir, err := os.MkdirTemp("", "rootless") - if err != nil { - return nil, err - } - driver.SetStateDir(stateDir) opt := &parent.Opt{ From 4b82b9ed28b087d1ef9cecf05142e0d6680f0bdd Mon Sep 17 00:00:00 2001 From: Edgar Lee Date: Sat, 27 Jan 2024 08:51:17 -0500 Subject: [PATCH 09/10] Expose rootless containerd socket directories for external access Signed-off-by: Edgar Lee (cherry picked from commit 0ac4c6a05682a104a8257c2b3b3e3a100431a54a) Signed-off-by: Brad Davidson --- pkg/rootless/mounts.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/rootless/mounts.go b/pkg/rootless/mounts.go index 2c19f2343cef..f4db388a9ede 100644 --- a/pkg/rootless/mounts.go +++ b/pkg/rootless/mounts.go @@ -6,6 +6,7 @@ package rootless import ( "fmt" "os" + "os/user" "path/filepath" "strings" @@ -25,11 +26,17 @@ func setupMounts(stateDir string) error { _ = os.RemoveAll(f) } + runDir, err := resolveRunDir() + if err != nil { + return err + } + mountMap := [][]string{ {"/var/log", filepath.Join(stateDir, "logs")}, {"/var/lib/cni", filepath.Join(stateDir, "cni")}, {"/var/lib/kubelet", filepath.Join(stateDir, "kubelet")}, {"/etc/rancher", filepath.Join(stateDir, "etc", "rancher")}, + {"/run/k3s/containerd", filepath.Join(runDir, "k3s", "containerd")}, } for _, v := range mountMap { @@ -91,3 +98,15 @@ func setupMount(target, dir string) error { logrus.Debug("Mounting ", dir, target, " none bind") return unix.Mount(dir, target, "none", unix.MS_BIND, "") } + +func resolveRunDir() (string, error) { + runDir := os.Getenv("XDG_RUNTIME_DIR") + if runDir == "" { + u, err := user.Lookup(os.Getenv("USER")) + if err != nil { + return "", err + } + runDir = filepath.Join("/run/user", u.Uid) + } + return runDir, nil +} From 6f0fecda9f629c85df309bdd5db035e35a02059c Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Fri, 2 Feb 2024 21:10:05 +0000 Subject: [PATCH 10/10] Bump kine and set NotifyInterval to what the apiserver expects Signed-off-by: Brad Davidson (cherry picked from commit de825845b2f1eca82c19892c327ed274abfa8901) Signed-off-by: Brad Davidson --- go.mod | 4 +-- go.sum | 8 +++--- pkg/cli/server/server.go | 1 + pkg/daemons/executor/executor.go | 36 +++++++++++++------------ pkg/etcd/etcd.go | 46 +++++++++++++++++--------------- 5 files changed, 51 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index 8e55d578a3b4..45a586796093 100644 --- a/go.mod +++ b/go.mod @@ -121,12 +121,12 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 github.com/json-iterator/go v1.1.12 github.com/k3s-io/helm-controller v0.15.8 - github.com/k3s-io/kine v0.11.0 + github.com/k3s-io/kine v0.11.4 github.com/klauspost/compress v1.17.2 github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000 github.com/lib/pq v1.10.2 github.com/libp2p/go-libp2p v0.30.0 - github.com/mattn/go-sqlite3 v1.14.17 + github.com/mattn/go-sqlite3 v1.14.19 github.com/minio/minio-go/v7 v7.0.33 github.com/mwitkow/go-http-dialer v0.0.0-20161116154839-378f744fb2b8 github.com/natefinch/lumberjack v2.0.0+incompatible diff --git a/go.sum b/go.sum index c92add086f30..98fc37f5ea5f 100644 --- a/go.sum +++ b/go.sum @@ -811,8 +811,8 @@ github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1 h1:B3039IkTPnwQEt4tIMjC6yd6b1Q3Z9ZZ github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1/go.mod h1:GgI1fQClQCFIzuVjlvdbMxNbnISt90gdfYyqiAIt65g= github.com/k3s-io/helm-controller v0.15.8 h1:CAMEPmiqf4ugUCpZdICGINthCn+hkG/l1fadn8aVjfQ= github.com/k3s-io/helm-controller v0.15.8/go.mod h1:AYitg40howLjKloL/zdjDDOPL1jg/K5R4af0tQcyPR8= -github.com/k3s-io/kine v0.11.0 h1:7tS0H9yBDxXiy1BgEEkBWLswwG/q4sARPTHdxOMz1qw= -github.com/k3s-io/kine v0.11.0/go.mod h1:tjSsWrCetgaGMTfnJW6vzqdT/qOPhF/+nUEaE+eixBA= +github.com/k3s-io/kine v0.11.4 h1:ZIXQT4vPPKNL9DwLF4dQ11tWtpJ1C/7OKNIpFmTkImo= +github.com/k3s-io/kine v0.11.4/go.mod h1:NmwOWsWgB3aScq5+LEYytAaceqkG7lmCLLjjrWug8v4= github.com/k3s-io/klog v1.0.0-k3s2 h1:yyvD2bQbxG7m85/pvNctLX2bUDmva5kOBvuZ77tTGBA= github.com/k3s-io/klog v1.0.0-k3s2/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= github.com/k3s-io/klog/v2 v2.80.1-k3s1 h1:mGMXURxxmabQurmtRhXuQTJ9jC0pvIhESSxRSymepS8= @@ -989,8 +989,8 @@ github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= -github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= +github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 25f48aaa8c42..6c391d0b536d 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -149,6 +149,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.ExtraEtcdArgs = cfg.ExtraEtcdArgs serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs serverConfig.ControlConfig.ClusterDomain = cfg.ClusterDomain + serverConfig.ControlConfig.Datastore.NotifyInterval = 5 * time.Second serverConfig.ControlConfig.Datastore.Endpoint = cfg.DatastoreEndpoint serverConfig.ControlConfig.Datastore.BackendTLSConfig.CAFile = cfg.DatastoreCAFile serverConfig.ControlConfig.Datastore.BackendTLSConfig.CertFile = cfg.DatastoreCertFile diff --git a/pkg/daemons/executor/executor.go b/pkg/daemons/executor/executor.go index 58dc832e402a..d28b5e062b0b 100644 --- a/pkg/daemons/executor/executor.go +++ b/pkg/daemons/executor/executor.go @@ -36,23 +36,25 @@ type Executor interface { } type ETCDConfig struct { - InitialOptions `json:",inline"` - Name string `json:"name,omitempty"` - ListenClientURLs string `json:"listen-client-urls,omitempty"` - ListenClientHTTPURLs string `json:"listen-client-http-urls,omitempty"` - ListenMetricsURLs string `json:"listen-metrics-urls,omitempty"` - ListenPeerURLs string `json:"listen-peer-urls,omitempty"` - AdvertiseClientURLs string `json:"advertise-client-urls,omitempty"` - DataDir string `json:"data-dir,omitempty"` - SnapshotCount int `json:"snapshot-count,omitempty"` - ServerTrust ServerTrust `json:"client-transport-security"` - PeerTrust PeerTrust `json:"peer-transport-security"` - ForceNewCluster bool `json:"force-new-cluster,omitempty"` - HeartbeatInterval int `json:"heartbeat-interval"` - ElectionTimeout int `json:"election-timeout"` - Logger string `json:"logger"` - LogOutputs []string `json:"log-outputs"` - ExperimentalInitialCorruptCheck bool `json:"experimental-initial-corrupt-check"` + InitialOptions `json:",inline"` + Name string `json:"name,omitempty"` + ListenClientURLs string `json:"listen-client-urls,omitempty"` + ListenClientHTTPURLs string `json:"listen-client-http-urls,omitempty"` + ListenMetricsURLs string `json:"listen-metrics-urls,omitempty"` + ListenPeerURLs string `json:"listen-peer-urls,omitempty"` + AdvertiseClientURLs string `json:"advertise-client-urls,omitempty"` + DataDir string `json:"data-dir,omitempty"` + SnapshotCount int `json:"snapshot-count,omitempty"` + ServerTrust ServerTrust `json:"client-transport-security"` + PeerTrust PeerTrust `json:"peer-transport-security"` + ForceNewCluster bool `json:"force-new-cluster,omitempty"` + HeartbeatInterval int `json:"heartbeat-interval"` + ElectionTimeout int `json:"election-timeout"` + Logger string `json:"logger"` + LogOutputs []string `json:"log-outputs"` + + ExperimentalInitialCorruptCheck bool `json:"experimental-initial-corrupt-check"` + ExperimentalWatchProgressNotifyInterval time.Duration `json:"experimental-watch-progress-notify-interval"` } type ServerTrust struct { diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 9a2e5fb01035..2ec3c8f544dd 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -908,13 +908,15 @@ func (e *ETCD) cluster(ctx context.Context, reset bool, options executor.Initial ClientCertAuth: true, TrustedCAFile: e.config.Runtime.ETCDPeerCA, }, - SnapshotCount: 10000, - ElectionTimeout: 5000, - HeartbeatInterval: 500, - Logger: "zap", - LogOutputs: []string{"stderr"}, - ExperimentalInitialCorruptCheck: true, - ListenClientHTTPURLs: e.listenClientHTTPURLs(), + SnapshotCount: 10000, + ElectionTimeout: 5000, + HeartbeatInterval: 500, + Logger: "zap", + LogOutputs: []string{"stderr"}, + ListenClientHTTPURLs: e.listenClientHTTPURLs(), + + ExperimentalInitialCorruptCheck: true, + ExperimentalWatchProgressNotifyInterval: e.config.Datastore.NotifyInterval, }, e.config.ExtraEtcdArgs) } @@ -967,20 +969,22 @@ func (e *ETCD) StartEmbeddedTemporary(ctx context.Context) error { embedded := executor.Embedded{} ctx, e.cancel = context.WithCancel(ctx) return embedded.ETCD(ctx, executor.ETCDConfig{ - InitialOptions: executor.InitialOptions{AdvertisePeerURL: peerURL}, - DataDir: tmpDataDir, - ForceNewCluster: true, - AdvertiseClientURLs: clientURL, - ListenClientURLs: clientURL, - ListenClientHTTPURLs: clientHTTPURL, - ListenPeerURLs: peerURL, - Logger: "zap", - HeartbeatInterval: 500, - ElectionTimeout: 5000, - SnapshotCount: 10000, - Name: e.name, - LogOutputs: []string{"stderr"}, - ExperimentalInitialCorruptCheck: true, + InitialOptions: executor.InitialOptions{AdvertisePeerURL: peerURL}, + DataDir: tmpDataDir, + ForceNewCluster: true, + AdvertiseClientURLs: clientURL, + ListenClientURLs: clientURL, + ListenClientHTTPURLs: clientHTTPURL, + ListenPeerURLs: peerURL, + Logger: "zap", + HeartbeatInterval: 500, + ElectionTimeout: 5000, + SnapshotCount: 10000, + Name: e.name, + LogOutputs: []string{"stderr"}, + + ExperimentalInitialCorruptCheck: true, + ExperimentalWatchProgressNotifyInterval: e.config.Datastore.NotifyInterval, }, append(e.config.ExtraEtcdArgs, "--max-snapshots=0", "--max-wals=0")) }