feat: Configurable retry backoff settings when retrying API calls (#4979

)

Signed-off-by: terrytangyuan <[email protected]>

util/env/env.go

@@ -2,6 +2,7 @@ package env
 
 import (
 	"os"
+	"strconv"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -19,3 +20,29 @@ func LookupEnvDurationOr(key string, o time.Duration) time.Duration {
 	}
 	return o
 }
+
+func LookupEnvIntOr(key string, o int) int {
+	v, found := os.LookupEnv(key)
+	if found {
+		d, err := strconv.Atoi(v)
+		if err != nil {
+			log.WithField(key, v).WithError(err).Panic("failed to convert to int")
+		} else {
+			return d
+		}
+	}
+	return o
+}
+
+func LookupEnvFloatOr(key string, o float64) float64 {
+	v, found := os.LookupEnv(key)
+	if found {
+		d, err := strconv.ParseFloat(v, 64)
+		if err != nil {
+			log.WithField(key, v).WithError(err).Panic("failed to convert to float")
+		} else {
+			return d
+		}
+	}
+	return o
+}

util/env/env_test.go

@@ -16,3 +16,21 @@ func TestLookupEnvDurationOr(t *testing.T) {
 	_ = os.Setenv("FOO", "1h")
 	assert.Equal(t, time.Hour, LookupEnvDurationOr("FOO", time.Second), "env var value")
 }
+
+func TestLookupEnvIntOr(t *testing.T) {
+	defer func() { _ = os.Unsetenv("FOO") }()
+	assert.Equal(t, 1, LookupEnvIntOr("", 1), "default value")
+	_ = os.Setenv("FOO", "not-int")
+	assert.Panics(t, func() { LookupEnvIntOr("FOO", 1) }, "bad value")
+	_ = os.Setenv("FOO", "2")
+	assert.Equal(t, 2, LookupEnvIntOr("FOO", 1), "env var value")
+}
+
+func TestLookupEnvFloatOr(t *testing.T) {
+	defer func() { _ = os.Unsetenv("FOO") }()
+	assert.Equal(t, 1., LookupEnvFloatOr("", 1.), "default value")
+	_ = os.Setenv("FOO", "not-float")
+	assert.Panics(t, func() { LookupEnvFloatOr("FOO", 1.) }, "bad value")
+	_ = os.Setenv("FOO", "2.0")
+	assert.Equal(t, 2., LookupEnvFloatOr("FOO", 1.), "env var value")
+}

util/errors/errors_test.go

@@ -74,4 +74,4 @@ func TestIsTransientErr(t *testing.T) {
 
 		_ = os.Unsetenv(transientEnvVarKey)
 	})
-}
+}

util/retry/retry.go

@@ -4,6 +4,8 @@ import (
 	"time"
 
 	"k8s.io/apimachinery/pkg/util/wait"
+
+	envutil "github.com/argoproj/argo/v2/util/env"
 )
 
 // DefaultRetry is a default retry backoff settings when retrying API calls
@@ -14,7 +16,7 @@ import (
 //     4      0.15
 //     5      0.31
 var DefaultRetry = wait.Backoff{
-	Steps:    5,
-	Duration: 10 * time.Millisecond,
-	Factor:   2,
+	Steps:    envutil.LookupEnvIntOr("RETRY_BACKOFF_STEPS", 5),
+	Duration: envutil.LookupEnvDurationOr("RETRY_BACKOFF_DURATION", 10*time.Millisecond),
+	Factor:   envutil.LookupEnvFloatOr("RETRY_BACKOFF_FACTOR", 2.),
 }