algod: Add static EnableTelemetry retry

cmd/algod/main.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
@@ -59,6 +60,11 @@ var sessionGUID = flag.String("s", "", "Telemetry Session GUID to use")
 var telemetryOverride = flag.String("t", "", `Override telemetry setting if supported (Use "true", "false", "0" or "1")`)
 var seed = flag.String("seed", "", "input to math/rand.Seed()")
 
+const (
+	defaultStaticTelemetryStartupTimeout = 5 * time.Second
+	defaultStaticTelemetryBGDialRetry    = 1 * time.Minute
+)
+
 func main() {
 	flag.Parse()
 	exitCode := run()
@@ -232,9 +238,28 @@ func run() int {
 					telemetryConfig.SessionGUID = *sessionGUID
 				}
 			}
-			err = log.EnableTelemetry(telemetryConfig)
+			// Try to enable remote telemetry now when URI is defined. Skip for DNS based telemetry.
+			ctx, telemetryCancelFn := context.WithTimeout(context.Background(), defaultStaticTelemetryStartupTimeout)
+			err = log.EnableTelemetryContext(ctx, telemetryConfig)
+			telemetryCancelFn()
 			if err != nil {
 				fmt.Fprintln(os.Stdout, "error creating telemetry hook", err)
+
+				// Remote telemetry init loop
+				go func() {
+					for {
+						time.Sleep(defaultStaticTelemetryBGDialRetry)
+						// Try to enable remote telemetry now when URI is defined. Skip for DNS based telemetry.
+						err := log.EnableTelemetryContext(context.Background(), telemetryConfig)
+						// Error occurs only if URI is defined and we need to retry later
+						if err == nil {
+							// Remote telemetry enabled or empty static URI, stop retrying
+							return
+						}
+						fmt.Fprintln(os.Stdout, "error creating telemetry hook", err)
+						// Try to reenable every minute
+					}
+				}()
 			}
 		}
 	}

cmd/algoh/main.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"os"
@@ -307,7 +308,7 @@ func initTelemetry(genesis bookkeeping.Genesis, log logging.Logger, dataDirector
 		telemetryConfig.Enable = logging.TelemetryOverride(*telemetryOverride, &telemetryConfig)
 
 		if telemetryConfig.Enable {
-			err = log.EnableTelemetry(telemetryConfig)
+			err = log.EnableTelemetryContext(context.Background(), telemetryConfig)
 			if err != nil {
 				fmt.Fprintln(os.Stdout, "error creating telemetry hook", err)
 				return

data/pools/transactionPool_test.go

@@ -19,6 +19,7 @@ package pools
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"fmt"
 	"math/rand"
 	"os"
@@ -1438,7 +1439,7 @@ func TestStateProofLogging(t *testing.T) {
 	// Set the logging to capture the telemetry Metrics into logging
 	logger := logging.TestingLog(t)
 	logger.SetLevel(logging.Info)
-	logger.EnableTelemetry(logging.TelemetryConfig{Enable: true, SendToLog: true})
+	logger.EnableTelemetryContext(context.Background(), logging.TelemetryConfig{Enable: true, SendToLog: true})
 	var buf bytes.Buffer
 	logger.SetOutput(&buf)
 

logging/log.go

@@ -35,6 +35,7 @@ logger.Info("New wallet was created")
 package logging
 
 import (
+	"context"
 	"io"
 	"runtime"
 	"runtime/debug"
@@ -154,7 +155,7 @@ type Logger interface {
 	// Adds a hook to the logger
 	AddHook(hook logrus.Hook)
 
-	EnableTelemetry(cfg TelemetryConfig) error
+	EnableTelemetryContext(ctx context.Context, cfg TelemetryConfig) error
 	UpdateTelemetryURI(uri string) error
 	GetTelemetryEnabled() bool
 	GetTelemetryUploadingEnabled() bool
@@ -389,11 +390,11 @@ func RegisterExitHandler(handler func()) {
 	logrus.RegisterExitHandler(handler)
 }
 
-func (l logger) EnableTelemetry(cfg TelemetryConfig) (err error) {
+func (l logger) EnableTelemetryContext(ctx context.Context, cfg TelemetryConfig) (err error) {
 	if l.loggerState.telemetry != nil || (!cfg.Enable && !cfg.SendToLog) {
 		return nil
 	}
-	return EnableTelemetry(cfg, &l)
+	return EnableTelemetryContext(ctx, cfg, &l)
 }
 
 func (l logger) UpdateTelemetryURI(uri string) (err error) {

logging/telemetry.go

@@ -17,6 +17,7 @@
 package logging
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"os"
@@ -36,8 +37,8 @@ const telemetrySeparator = "/"
 const logBufferDepth = 2
 
 // EnableTelemetry configures and enables telemetry based on the config provided
-func EnableTelemetry(cfg TelemetryConfig, l *logger) (err error) {
-	telemetry, err := makeTelemetryState(cfg, createElasticHook)
+func EnableTelemetryContext(ctx context.Context, cfg TelemetryConfig, l *logger) (err error) {
+	telemetry, err := makeTelemetryStateContext(ctx, cfg, createElasticHookContext)
 	if err != nil {
 		return
 	}
@@ -70,14 +71,14 @@ func makeLevels(min logrus.Level) []logrus.Level {
 	return levels
 }
 
-func makeTelemetryState(cfg TelemetryConfig, hookFactory hookFactory) (*telemetryState, error) {
+func makeTelemetryStateContext(ctx context.Context, cfg TelemetryConfig, hookFactory hookFactory) (*telemetryState, error) {
 	telemetry := &telemetryState{}
 	telemetry.history = createLogBuffer(logBufferDepth)
 	if cfg.Enable {
 		if cfg.SessionGUID == "" {
 			cfg.SessionGUID = uuid.New()
 		}
-		hook, err := createTelemetryHook(cfg, telemetry.history, hookFactory)
+		hook, err := createTelemetryHookContext(ctx, cfg, telemetry.history, hookFactory)
 		if err != nil {
 			return nil, err
 		}

logging/telemetryCommon.go

@@ -17,9 +17,11 @@
 package logging
 
 import (
+	"context"
+	"sync"
+
 	"github.com/algorand/go-deadlock"
 	"github.com/sirupsen/logrus"
-	"sync"
 )
 
 type telemetryHook interface {
@@ -81,4 +83,4 @@ type asyncTelemetryHook struct {
 // A dummy noop type to get rid of checks like telemetry.hook != nil
 type dummyHook struct{}
 
-type hookFactory func(cfg TelemetryConfig) (logrus.Hook, error)
+type hookFactory func(ctx context.Context, cfg TelemetryConfig) (logrus.Hook, error)

logging/telemetry_test.go

@@ -17,13 +17,15 @@
 package logging
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/require"
 	"os"
 	"testing"
 
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+
 	"github.com/algorand/go-deadlock"
 
 	"github.com/algorand/go-algorand/config"
@@ -84,7 +86,7 @@ func makeTelemetryTestFixtureWithConfig(minLevel logrus.Level, cfg *TelemetryCon
 	f.l = Base().(logger)
 	f.l.SetLevel(Debug) // Ensure logging doesn't filter anything out
 
-	f.telem, _ = makeTelemetryState(lcfg, func(cfg TelemetryConfig) (hook logrus.Hook, err error) {
+	f.telem, _ = makeTelemetryStateContext(context.Background(), lcfg, func(ctx context.Context, cfg TelemetryConfig) (hook logrus.Hook, err error) {
 		return &f.hook, nil
 	})
 	f.l.loggerState.telemetry = f.telem
@@ -138,7 +140,7 @@ func TestCreateHookError(t *testing.T) {
 
 	cfg := createTelemetryConfig()
 	cfg.Enable = true
-	telem, err := makeTelemetryState(cfg, func(cfg TelemetryConfig) (hook logrus.Hook, err error) {
+	telem, err := makeTelemetryStateContext(context.Background(), cfg, func(ctx context.Context, cfg TelemetryConfig) (hook logrus.Hook, err error) {
 		return nil, fmt.Errorf("failed")
 	})
 

logging/telemetryhook.go

@@ -17,6 +17,7 @@
 package logging
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/olivere/elastic"
@@ -228,14 +229,14 @@ func (el elasticClientLogger) Printf(format string, v ...interface{}) {
 	}
 }
 
-func createElasticHook(cfg TelemetryConfig) (hook logrus.Hook, err error) {
+func createElasticHookContext(ctx context.Context, cfg TelemetryConfig) (hook logrus.Hook, err error) {
 	// Returning an error here causes issues... need the hooks to be created even if the elastic hook fails so that
 	// things can recover later.
 	if cfg.URI == "" {
 		return nil, nil
 	}
 
-	client, err := elastic.NewClient(elastic.SetURL(cfg.URI),
+	client, err := elastic.DialContext(ctx, elastic.SetURL(cfg.URI),
 		elastic.SetBasicAuth(cfg.UserName, cfg.Password),
 		elastic.SetSniff(false),
 		elastic.SetGzip(true),
@@ -257,12 +258,12 @@ func createElasticHook(cfg TelemetryConfig) (hook logrus.Hook, err error) {
 }
 
 // createTelemetryHook creates the Telemetry log hook, or returns nil if remote logging is not enabled
-func createTelemetryHook(cfg TelemetryConfig, history *logBuffer, hookFactory hookFactory) (hook logrus.Hook, err error) {
+func createTelemetryHookContext(ctx context.Context, cfg TelemetryConfig, history *logBuffer, hookFactory hookFactory) (hook logrus.Hook, err error) {
 	if !cfg.Enable {
 		return nil, fmt.Errorf("createTelemetryHook called when telemetry not enabled")
 	}
 
-	hook, err = hookFactory(cfg)
+	hook, err = hookFactory(ctx, cfg)
 
 	if err != nil {
 		return nil, err
@@ -290,7 +291,8 @@ func (hook *asyncTelemetryHook) UpdateHookURI(uri string) (err error) {
 		copy := tfh.telemetryConfig
 		copy.URI = uri
 		var newHook logrus.Hook
-		newHook, err = tfh.factory(copy)
+
+		newHook, err = tfh.factory(context.Background(), copy)
 
 		if err == nil && newHook != nil {
 			tfh.wrappedHook = newHook