diff --git a/cd/manager/models.go b/cd/manager/models.go index 82dc9b2..43173ae 100644 --- a/cd/manager/models.go +++ b/cd/manager/models.go @@ -10,6 +10,7 @@ import ( const DefaultTick = 10 * time.Second const DefaultTtlDays = 1 const DefaultHttpWaitTime = 30 * time.Second +const DefaultHttpRetries = 3 const DefaultWaitTime = 5 * time.Minute type EnvType string diff --git a/cd/manager/notifs/deploy.go b/cd/manager/notifs/deploy.go index 42d03b6..6bc8a3a 100644 --- a/cd/manager/notifs/deploy.go +++ b/cd/manager/notifs/deploy.go @@ -61,13 +61,17 @@ func (d deployNotif) getTitle() string { } else if manual, _ := d.state.Params[job.DeployJobParam_Manual].(bool); manual { qualifier = job.DeployJobParam_Manual } + prettyStage := string(d.state.Stage) + if d.state.Stage == job.JobStage_Dequeued { + prettyStage = prettyStageDequeued + } return fmt.Sprintf( "3Box Labs `%s` %s %s %s %s", envName(d.env), strings.ToUpper(component), cases.Title(language.English).String(qualifier), "Deployment", - strings.ToUpper(string(d.state.Stage)), + strings.ToUpper(prettyStage), ) } diff --git a/cd/manager/notifs/discord.go b/cd/manager/notifs/discord.go index 361be12..89ed162 100644 --- a/cd/manager/notifs/discord.go +++ b/cd/manager/notifs/discord.go @@ -44,6 +44,9 @@ const discordPacing = 2 * time.Second const shaTagLength = 12 +// Show "queued" for "dequeued" jobs to make it more understandable +const prettyStageDequeued = "queued" + var _ manager.Notifs = &JobNotifs{} type JobNotifs struct { diff --git a/cd/manager/notifs/workflow.go b/cd/manager/notifs/workflow.go index 2dfec0d..cf5b177 100644 --- a/cd/manager/notifs/workflow.go +++ b/cd/manager/notifs/workflow.go @@ -51,7 +51,11 @@ func (w workflowNotif) getTitle() string { if workflowName, found := w.state.Params[job.WorkflowJobParam_Name].(string); found { jobName = workflowName } - return fmt.Sprintf("%s %s", jobName, strings.ToUpper(string(w.state.Stage))) + prettyStage := string(w.state.Stage) + if w.state.Stage == job.JobStage_Dequeued { + prettyStage = prettyStageDequeued + } + return fmt.Sprintf("%s %s", jobName, strings.ToUpper(prettyStage)) } func (w workflowNotif) getFields() []discord.EmbedField { diff --git a/cd/manager/repository/github.go b/cd/manager/repository/github.go index 5c9baea..a586173 100644 --- a/cd/manager/repository/github.go +++ b/cd/manager/repository/github.go @@ -211,13 +211,16 @@ func (g Github) CheckWorkflowStatus(workflow job.Workflow, workflowRunId int64) } func (g Github) getWorkflowRun(org, repo string, workflowRunId int64) (*github.WorkflowRun, error) { - ctx, cancel := context.WithTimeout(context.Background(), manager.DefaultHttpWaitTime) - defer cancel() - - if workflowRun, resp, err := g.client.Actions.GetWorkflowRunByID(ctx, org, repo, workflowRunId); err != nil { - return nil, err - } else { - log.Printf("getWorkflowRun: run=%s, rate limit=%d, remaining=%d, resetAt=%s", workflowRun.GetHTMLURL(), resp.Rate.Limit, resp.Rate.Remaining, resp.Rate.Reset) - return workflowRun, nil - } + return manager.RetryWithResultAndError[*github.WorkflowRun]( + context.Background(), + manager.DefaultHttpWaitTime, + manager.DefaultHttpRetries, + func(ctx context.Context, _ ...interface{}) (*github.WorkflowRun, error) { + if workflowRun, resp, err := g.client.Actions.GetWorkflowRunByID(ctx, org, repo, workflowRunId); err != nil { + return nil, err + } else { + log.Printf("getWorkflowRun: run=%s, rate limit=%d, remaining=%d, resetAt=%s", workflowRun.GetHTMLURL(), resp.Rate.Limit, resp.Rate.Remaining, resp.Rate.Reset) + return workflowRun, nil + } + }) } diff --git a/cd/manager/utils.go b/cd/manager/utils.go index 07e086b..202e2d8 100644 --- a/cd/manager/utils.go +++ b/cd/manager/utils.go @@ -1,6 +1,7 @@ package manager import ( + "context" "encoding/json" "fmt" "regexp" @@ -76,3 +77,33 @@ func AdvanceJob(jobState job.JobState, jobStage job.JobStage, ts time.Time, err } return jobState, err } + +func RetryWithResultAndError[R any](parentCtx context.Context, timeout time.Duration, numRetries int, fn func(context.Context, ...interface{}) (R, error), args ...interface{}) (R, error) { + retry := func() (R, error) { + ctx, cancel := context.WithTimeout(parentCtx, timeout) + defer cancel() + + return fn(ctx, args) + } + for i := 0; i < numRetries-1; i++ { + if res, err := retry(); err != context.DeadlineExceeded { + return res, err + } + } + return retry() +} + +func RetryWithError(parentCtx context.Context, timeout time.Duration, numRetries int, fn func(context.Context, ...interface{}) error, args ...interface{}) error { + retry := func() error { + ctx, cancel := context.WithTimeout(parentCtx, timeout) + defer cancel() + + return fn(ctx, args) + } + for i := 0; i < numRetries-1; i++ { + if err := retry(); err != context.DeadlineExceeded { + return err + } + } + return retry() +}