Log TLS failures when initializing the backend (#3690)

CHANGELOG.md

@@ -13,9 +13,14 @@ Versioning](http://semver.org/spec/v2.0.0.html).
 - [Web] Added the ability for labels and annotations with links to images to be
 displayed inline.
 - [Web] Added additional modes for those with colour blindness.
+- Added a `timeout` flag to `sensu-backend init`.
+
 ### Changed
 - Removed deprecated flags in `sensuctl silenced update` subcommand.
 
+### Fixed
+- `sensu-backend init` now logs any TLS failures encountered.
+
 ## [5.19.1] - 2020-04-13
 
 ### Fixed

backend/backend.go

@@ -238,8 +238,8 @@ func Initialize(ctx context.Context, config *Config) (*Backend, error) {
 
 	// Initialize pipelined
 	pipeline, err := pipelined.New(pipelined.Config{
-		Store: stor,
-		Bus:   bus,
+		Store:                   stor,
+		Bus:                     bus,
 		ExtensionExecutorGetter: rpc.NewGRPCExtensionExecutor,
 		AssetGetter:             assetGetter,
 		BufferSize:              viper.GetInt(FlagPipelinedBufferSize),
@@ -313,14 +313,14 @@ func Initialize(ctx context.Context, config *Config) (*Backend, error) {
 	// Initialize keepalived
 	keepalive, err := keepalived.New(keepalived.Config{
 		DeregistrationHandler: config.DeregistrationHandler,
-		Bus:             bus,
-		Store:           stor,
-		EventStore:      eventStoreProxy,
-		LivenessFactory: liveness.EtcdFactory(b.runCtx, b.Client),
-		RingPool:        ringPool,
-		BufferSize:      viper.GetInt(FlagKeepalivedBufferSize),
-		WorkerCount:     viper.GetInt(FlagKeepalivedWorkers),
-		StoreTimeout:    2 * time.Minute,
+		Bus:                   bus,
+		Store:                 stor,
+		EventStore:            eventStoreProxy,
+		LivenessFactory:       liveness.EtcdFactory(b.runCtx, b.Client),
+		RingPool:              ringPool,
+		BufferSize:            viper.GetInt(FlagKeepalivedBufferSize),
+		WorkerCount:           viper.GetInt(FlagKeepalivedWorkers),
+		StoreTimeout:          2 * time.Minute,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("error initializing %s: %s", keepalive.Name(), err)

backend/cmd/init.go

@@ -15,18 +15,21 @@ import (
 	etcdstore "github.com/sensu/sensu-go/backend/store/etcd"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"google.golang.org/grpc"
 )
 
 const (
+	defaultTimeout = "5"
+
 	flagInitAdminUsername = "cluster-admin-username"
 	flagInitAdminPassword = "cluster-admin-password"
 	flagInteractive       = "interactive"
+	flagTimeout           = "timeout"
 )
 
 type seedConfig struct {
 	backend.Config
 	SeedConfig seeds.Config
+	Timeout    time.Duration
 }
 
 type initOpts struct {
@@ -115,13 +118,12 @@ func InitCommand() *cobra.Command {
 				clientURLs = viper.GetStringSlice(flagEtcdAdvertiseClientURLs)
 			}
 
+			timeout := viper.GetDuration(flagTimeout)
+
 			client, err := clientv3.New(clientv3.Config{
 				Endpoints:   clientURLs,
-				DialTimeout: 5 * time.Second,
+				DialTimeout: timeout * time.Second,
 				TLS:         tlsConfig,
-				DialOptions: []grpc.DialOption{
-					grpc.WithBlock(),
-				},
 			})
 
 			if err != nil {
@@ -150,15 +152,37 @@ func InitCommand() *cobra.Command {
 					AdminUsername: uname,
 					AdminPassword: pword,
 				},
+				Timeout: timeout,
+			}
+
+			// Make sure at least one of the provided endpoints is reachable. This is
+			// required to debug TLS errors because the seeding below will not print
+			// the latest connection error (see
+			// https://github.com/sensu/sensu-go/issues/3663)
+			for _, url := range clientURLs {
+				tctx, cancel := context.WithTimeout(context.Background(), timeout*time.Second)
+				defer cancel()
+				_, err = client.Status(tctx, url)
+				if err != nil {
+					// We do not need to log the error, etcd's client interceptor will log
+					// the actual underlying error
+					continue
+				}
+				// The endpoint did not return any error, therefore we can proceed
+				goto seed
 			}
+			// All endpoints returned an error, return the latest one
+			return err
 
+		seed:
 			return seedCluster(client, seedConfig)
 		},
 	}
 
 	cmd.Flags().String(flagInitAdminUsername, "", "cluster admin username")
 	cmd.Flags().String(flagInitAdminPassword, "", "cluster admin password")
 	cmd.Flags().Bool(flagInteractive, false, "interactive mode")
+	cmd.Flags().String(flagTimeout, defaultTimeout, "timeout, in seconds, for failing to establish a connection to etcd")
 
 	setupErr = handleConfig(cmd, false)
 
@@ -167,7 +191,7 @@ func InitCommand() *cobra.Command {
 
 func seedCluster(client *clientv3.Client, config seedConfig) error {
 	store := etcdstore.NewStore(client, "")
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	ctx, cancel := context.WithTimeout(context.Background(), config.Timeout*time.Second)
 	defer cancel()
 	if err := seeds.SeedCluster(ctx, store, config.SeedConfig); err != nil {
 		return err

backend/cmd/start.go

@@ -250,7 +250,6 @@ func StartCommand(initialize InitializeFunc) *cobra.Command {
 					log.Println(http.ListenAndServe("127.0.0.1:6060", nil))
 				}()
 			}
-
 			return sensuBackend.RunWithInitializer(initialize)
 		},
 	}