feat: add retries for 3rd party api calls

cd/manager/models.go

@@ -10,6 +10,7 @@ import (
 const DefaultTick = 10 * time.Second
 const DefaultTtlDays = 1
 const DefaultHttpWaitTime = 30 * time.Second
+const DefaultHttpRetries = 3
 const DefaultWaitTime = 5 * time.Minute
 
 type EnvType string

cd/manager/notifs/deploy.go

@@ -61,13 +61,17 @@ func (d deployNotif) getTitle() string {
 	} else if manual, _ := d.state.Params[job.DeployJobParam_Manual].(bool); manual {
 		qualifier = job.DeployJobParam_Manual
 	}
+	prettyStage := string(d.state.Stage)
+	if d.state.Stage == job.JobStage_Dequeued {
+		prettyStage = prettyStageDequeued
+	}
 	return fmt.Sprintf(
 		"3Box Labs `%s` %s %s %s %s",
 		envName(d.env),
 		strings.ToUpper(component),
 		cases.Title(language.English).String(qualifier),
 		"Deployment",
-		strings.ToUpper(string(d.state.Stage)),
+		strings.ToUpper(prettyStage),
 	)
 }
 

cd/manager/notifs/discord.go

@@ -44,6 +44,9 @@ const discordPacing = 2 * time.Second
 
 const shaTagLength = 12
 
+// Show "queued" for "dequeued" jobs to make it more understandable
+const prettyStageDequeued = "queued"
+
 var _ manager.Notifs = &JobNotifs{}
 
 type JobNotifs struct {

cd/manager/notifs/workflow.go

@@ -51,7 +51,11 @@ func (w workflowNotif) getTitle() string {
 	if workflowName, found := w.state.Params[job.WorkflowJobParam_Name].(string); found {
 		jobName = workflowName
 	}
-	return fmt.Sprintf("%s %s", jobName, strings.ToUpper(string(w.state.Stage)))
+	prettyStage := string(w.state.Stage)
+	if w.state.Stage == job.JobStage_Dequeued {
+		prettyStage = prettyStageDequeued
+	}
+	return fmt.Sprintf("%s %s", jobName, strings.ToUpper(prettyStage))
 }
 
 func (w workflowNotif) getFields() []discord.EmbedField {

cd/manager/repository/github.go

@@ -211,13 +211,16 @@ func (g Github) CheckWorkflowStatus(workflow job.Workflow, workflowRunId int64)
 }
 
 func (g Github) getWorkflowRun(org, repo string, workflowRunId int64) (*github.WorkflowRun, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), manager.DefaultHttpWaitTime)
-	defer cancel()
-
-	if workflowRun, resp, err := g.client.Actions.GetWorkflowRunByID(ctx, org, repo, workflowRunId); err != nil {
-		return nil, err
-	} else {
-		log.Printf("getWorkflowRun: run=%s, rate limit=%d, remaining=%d, resetAt=%s", workflowRun.GetHTMLURL(), resp.Rate.Limit, resp.Rate.Remaining, resp.Rate.Reset)
-		return workflowRun, nil
-	}
+	return manager.RetryWithResultAndError[*github.WorkflowRun](
+		context.Background(),
+		manager.DefaultHttpWaitTime,
+		manager.DefaultHttpRetries,
+		func(ctx context.Context, _ ...interface{}) (*github.WorkflowRun, error) {
+			if workflowRun, resp, err := g.client.Actions.GetWorkflowRunByID(ctx, org, repo, workflowRunId); err != nil {
+				return nil, err
+			} else {
+				log.Printf("getWorkflowRun: run=%s, rate limit=%d, remaining=%d, resetAt=%s", workflowRun.GetHTMLURL(), resp.Rate.Limit, resp.Rate.Remaining, resp.Rate.Reset)
+				return workflowRun, nil
+			}
+		})
 }

cd/manager/utils.go

@@ -1,6 +1,7 @@
 package manager
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"regexp"
@@ -76,3 +77,33 @@ func AdvanceJob(jobState job.JobState, jobStage job.JobStage, ts time.Time, err
 	}
 	return jobState, err
 }
+
+func RetryWithResultAndError[R any](parentCtx context.Context, timeout time.Duration, numRetries int, fn func(context.Context, ...interface{}) (R, error), args ...interface{}) (R, error) {
+	retry := func() (R, error) {
+		ctx, cancel := context.WithTimeout(parentCtx, timeout)
+		defer cancel()
+
+		return fn(ctx, args)
+	}
+	for i := 0; i < numRetries-1; i++ {
+		if res, err := retry(); err != context.DeadlineExceeded {
+			return res, err
+		}
+	}
+	return retry()
+}
+
+func RetryWithError(parentCtx context.Context, timeout time.Duration, numRetries int, fn func(context.Context, ...interface{}) error, args ...interface{}) error {
+	retry := func() error {
+		ctx, cancel := context.WithTimeout(parentCtx, timeout)
+		defer cancel()
+
+		return fn(ctx, args)
+	}
+	for i := 0; i < numRetries-1; i++ {
+		if err := retry(); err != context.DeadlineExceeded {
+			return err
+		}
+	}
+	return retry()
+}