etcd: use a separate connection to check peer versions

There is a data race in etcd that breaks the internal state in etcd client
implementation for some server setups (user/pass authentication with
JWTs).

Author: Andrew Lytvynov

lib/backend/etcdbk/etcd.go

@@ -216,11 +216,10 @@ func New(ctx context.Context, params backend.Params) (*EtcdBackend, error) {
 		buf:              buf,
 	}
 
+	// Check that the etcd nodes are at least the minimum version supported
 	if err = b.reconnect(ctx); err != nil {
 		return nil, trace.Wrap(err)
 	}
-
-	// Check that the etcd nodes are at least the minimum version supported
 	timeout, cancel := context.WithTimeout(ctx, time.Second*3*time.Duration(len(cfg.Nodes)))
 	defer cancel()
 	for _, n := range cfg.Nodes {
@@ -237,6 +236,13 @@ func New(ctx context.Context, params backend.Params) (*EtcdBackend, error) {
 		}
 	}
 
+	// Reconnect the etcd client to work around a data race in their code.
+	// Upstream fix: https://github.com/etcd-io/etcd/pull/12992
+	if err = b.reconnect(ctx); err != nil {
+		return nil, trace.Wrap(err)
+	}
+	go b.asyncWatch()
+
 	// Wrap backend in a input sanitizer and return it.
 	return b, nil
 }
@@ -292,6 +298,12 @@ func (b *EtcdBackend) CloseWatchers() {
 }
 
 func (b *EtcdBackend) reconnect(ctx context.Context) error {
+	if b.client != nil {
+		if err := b.client.Close(); err != nil {
+			b.Entry.WithError(err).Warningf("Failed closing existing etcd client on reconnect.")
+		}
+	}
+
 	tlsConfig := utils.TLSConfig(nil)
 
 	if b.cfg.TLSCertFile != "" {
@@ -340,7 +352,6 @@ func (b *EtcdBackend) reconnect(ctx context.Context) error {
 		return trace.Wrap(err)
 	}
 	b.client = clt
-	go b.asyncWatch()
 	return nil
 }