diff --git a/cd/manager/common/job/models.go b/cd/manager/common/job/models.go index c55f5ba..b1cbfa8 100644 --- a/cd/manager/common/job/models.go +++ b/cd/manager/common/job/models.go @@ -32,11 +32,11 @@ const ( ) const ( - JobParam_Id string = "id" - JobParam_Error string = "error" - JobParam_Elapsed string = "elapsed" - JobParam_Start string = "start" - JobParam_Source string = "source" + JobParam_Id string = "id" + JobParam_Error string = "error" + JobParam_WaitTime string = "waitTime" + JobParam_Start string = "start" + JobParam_Source string = "source" ) const ( diff --git a/cd/manager/jobmanager/jobManager.go b/cd/manager/jobmanager/jobManager.go index 0f49083..9213741 100644 --- a/cd/manager/jobmanager/jobManager.go +++ b/cd/manager/jobmanager/jobManager.go @@ -169,10 +169,10 @@ func (m *JobManager) processJobs() { // Decide how to proceed based on the first job from the list if dequeuedJobs[0].Type == job.JobType_Deploy { m.processDeployJobs(dequeuedJobs) - // There are two scenarios for anchor jobs on encountering a deploy job at the head of the queue: - // - Anchor jobs are started if no deployment was *started*, even if this deploy job was ahead of + // There are two scenarios for anchor jobs on encountering a deployment job at the head of the queue: + // - Anchor jobs are started if no deployment was *started*, even if this deployment job was ahead of // anchor jobs in the queue. - // - Anchor jobs are not started since a deploy job was *dequeued* ahead of them. (This would be the + // - Anchor jobs are not started since a deployment job was *dequeued* ahead of them. (This would be the // normal behavior for a job queue, i.e. jobs get processed in the order they were scheduled.) // // The first scenario only applies to the QA environment that is used for running the E2E tests. E2E diff --git a/cd/manager/notifs/anchor.go b/cd/manager/notifs/anchor.go index 423bb91..a776930 100644 --- a/cd/manager/notifs/anchor.go +++ b/cd/manager/notifs/anchor.go @@ -2,11 +2,13 @@ package notifs import ( "fmt" + "os" "strings" "github.com/disgoorg/disgo/discord" "github.com/disgoorg/disgo/webhook" + "github.com/3box/pipeline-tools/cd/manager" "github.com/3box/pipeline-tools/cd/manager/common/job" ) @@ -16,6 +18,8 @@ type anchorNotif struct { state job.JobState alertWebhook webhook.Client warningWebhook webhook.Client + region string + env string } func newAnchorNotif(jobState job.JobState) (jobNotif, error) { @@ -24,7 +28,13 @@ func newAnchorNotif(jobState job.JobState) (jobNotif, error) { } else if w, err := parseDiscordWebhookUrl("DISCORD_WARNING_WEBHOOK"); err != nil { return nil, err } else { - return &anchorNotif{jobState, a, w}, nil + return &anchorNotif{ + jobState, + a, + w, + os.Getenv("AWS_REGION"), + os.Getenv(manager.EnvVar_Env), + }, nil } } @@ -69,3 +79,17 @@ func (a anchorNotif) getColor() discordColor { } return colorForStage(a.state.Stage) } + +func (a anchorNotif) getUrl() string { + if taskId, found := a.state.Params[job.JobParam_Id].(string); found { + idParts := strings.Split(taskId, "/") + return fmt.Sprintf( + "https://%s.console.aws.amazon.com/cloudwatch/home?region=%s#logsV2:log-groups/log-group/$252Fecs$252Fceramic-%s-cas/log-events/cas_anchor$252Fcas_anchor$252F%s", + a.region, + a.region, + a.env, + idParts[len(idParts)-1], + ) + } + return "" +} diff --git a/cd/manager/notifs/deploy.go b/cd/manager/notifs/deploy.go index 10186d8..42d03b6 100644 --- a/cd/manager/notifs/deploy.go +++ b/cd/manager/notifs/deploy.go @@ -93,3 +93,7 @@ func envName(env manager.EnvType) string { return "" } } + +func (d deployNotif) getUrl() string { + return "" +} diff --git a/cd/manager/notifs/discord.go b/cd/manager/notifs/discord.go index c6c19b8..361be12 100644 --- a/cd/manager/notifs/discord.go +++ b/cd/manager/notifs/discord.go @@ -37,6 +37,7 @@ const ( notifField_TestE2E string = "E2E Tests" notifField_TestSmoke string = "Smoke Tests" notifField_Workflow string = "Workflow(s)" + notifField_Logs string = "Logs" ) const discordPacing = 2 * time.Second @@ -56,6 +57,7 @@ type jobNotif interface { getTitle() string getFields() []discord.EmbedField getColor() discordColor + getUrl() string } func NewJobNotifs(db manager.Database, cache manager.Cache) (manager.Notifs, error) { @@ -152,21 +154,22 @@ func (n JobNotifs) getNotifFields(jobState job.JobState) []discord.EmbedField { Value: deployTags, }) } - // If the job just started, also add the wait time. + // If the job just started, also display the queue wait time. if jobState.Stage == job.JobStage_Started { - if elapsedTime, found := jobState.Params[job.JobParam_Elapsed].(string); found { - parsedElapsedTime, _ := time.ParseDuration(elapsedTime) - waitTime := formattedDuration(parsedElapsedTime) - if len(waitTime) > 0 { + if waitTime, found := jobState.Params[job.JobParam_WaitTime].(string); found { + parsedWaitTime, _ := time.ParseDuration(waitTime) + prettyWaitTime := prettyDuration(parsedWaitTime) + if len(prettyWaitTime) > 0 { fields = append(fields, discord.EmbedField{ Name: notifField_WaitTime, - Value: waitTime, + Value: prettyWaitTime, }) } } - } + } else + // Only need to display the run time once the job progresses beyond the "started" stage if startTime, found := jobState.Params[job.JobParam_Start].(float64); found { - runTime := formattedDuration(time.Since(time.Unix(0, int64(startTime)))) + runTime := prettyDuration(time.Since(time.Unix(0, int64(startTime)))) if len(runTime) > 0 { fields = append(fields, discord.EmbedField{ Name: notifField_RunTime, @@ -178,6 +181,14 @@ func (n JobNotifs) getNotifFields(jobState job.JobState) []discord.EmbedField { if activeJobs := n.getActiveJobs(jobState); len(activeJobs) > 0 { fields = append(fields, activeJobs...) } + if jn, err := n.getJobNotif(jobState); err == nil { + if len(jn.getUrl()) > 0 { + fields = append(fields, discord.EmbedField{ + Name: notifField_Logs, + Value: fmt.Sprintf("[%s](%s)", jobState.JobId, jn.getUrl()), + }) + } + } return fields } @@ -259,7 +270,11 @@ func (n JobNotifs) getActiveJobsByType(jobState job.JobState, jobType job.JobTyp for _, activeJob := range activeJobs { // Exclude job for which this notification is being generated if activeJob.JobId != jobState.JobId { - message += fmt.Sprintf("%s (%s)\n", activeJob.JobId, activeJob.Stage) + if jn, err := n.getJobNotif(activeJob); (err == nil) && (len(jn.getUrl()) > 0) { + message += fmt.Sprintf("[%s](%s)\n", activeJob.JobId, jn.getUrl()) + } else { + message += activeJob.JobId + "\n" + } } } return discord.EmbedField{ @@ -307,7 +322,7 @@ func colorForStage(jobStage job.JobStage) discordColor { } } -func formattedDuration(duration time.Duration) string { +func prettyDuration(duration time.Duration) string { hours := int(duration.Seconds() / 3600) minutes := int(duration.Seconds()/60) % 60 seconds := int(duration.Seconds()) % 60 diff --git a/cd/manager/notifs/e2e.go b/cd/manager/notifs/e2e.go index 4e8d5ba..bfcd9b6 100644 --- a/cd/manager/notifs/e2e.go +++ b/cd/manager/notifs/e2e.go @@ -35,3 +35,7 @@ func (e e2eTestNotif) getFields() []discord.EmbedField { func (e e2eTestNotif) getColor() discordColor { return colorForStage(e.state.Stage) } + +func (e e2eTestNotif) getUrl() string { + return "" +} diff --git a/cd/manager/notifs/smoke.go b/cd/manager/notifs/smoke.go index 4992234..04321da 100644 --- a/cd/manager/notifs/smoke.go +++ b/cd/manager/notifs/smoke.go @@ -2,6 +2,8 @@ package notifs import ( "fmt" + "github.com/3box/pipeline-tools/cd/manager" + "os" "strings" "github.com/disgoorg/disgo/discord" @@ -13,11 +15,13 @@ import ( var _ jobNotif = &smokeTestNotif{} type smokeTestNotif struct { - state job.JobState + state job.JobState + region string + env string } func newSmokeTestNotif(jobState job.JobState) (jobNotif, error) { - return &smokeTestNotif{jobState}, nil + return &smokeTestNotif{jobState, os.Getenv("AWS_REGION"), os.Getenv(manager.EnvVar_Env)}, nil } func (s smokeTestNotif) getChannels() []webhook.Client { @@ -35,3 +39,17 @@ func (s smokeTestNotif) getFields() []discord.EmbedField { func (s smokeTestNotif) getColor() discordColor { return colorForStage(s.state.Stage) } + +func (s smokeTestNotif) getUrl() string { + if taskId, found := s.state.Params[job.JobParam_Id].(string); found { + idParts := strings.Split(taskId, "/") + return fmt.Sprintf( + "https://%s.console.aws.amazon.com/cloudwatch/home?region=%s#logsV2:log-groups/log-group/$252Fecs$252Fceramic-qa-tests/log-events/%s-smoke-tests$252Fsmoke$252F%s", + s.region, + s.region, + s.env, + idParts[len(idParts)-1], + ) + } + return "" +} diff --git a/cd/manager/notifs/workflow.go b/cd/manager/notifs/workflow.go index f651f38..2dfec0d 100644 --- a/cd/manager/notifs/workflow.go +++ b/cd/manager/notifs/workflow.go @@ -2,7 +2,6 @@ package notifs import ( "fmt" - "strconv" "strings" "github.com/disgoorg/disgo/discord" @@ -17,7 +16,6 @@ const defaultWorkflowJobName = "Workflow" const ( workflowNotifField_Branch = "Branch" workflowNotifField_TestSelector = "Test Selector" - workflowNotifField_Logs = "Logs" ) const ( workflowTestSelector_Wildcard = "." @@ -77,18 +75,13 @@ func (w workflowNotif) getFields() []discord.EmbedField { }, ) } - if len(w.workflow.Url) > 0 { - notifFields = append( - notifFields, - discord.EmbedField{ - Name: workflowNotifField_Logs, - Value: fmt.Sprintf("[%s (%s)](%s)", w.workflow.Repo, strconv.Itoa(int(w.workflow.Id)), w.workflow.Url), - }, - ) - } return notifFields } func (w workflowNotif) getColor() discordColor { return colorForStage(w.state.Stage) } + +func (w workflowNotif) getUrl() string { + return w.workflow.Url +} diff --git a/cd/manager/utils.go b/cd/manager/utils.go index 4feea87..07e086b 100644 --- a/cd/manager/utils.go +++ b/cd/manager/utils.go @@ -58,8 +58,14 @@ func IsV5WorkerJob(jobState job.JobState) bool { // AdvanceJob will move a JobState to a new JobStage in the Database and send an appropriate notification func AdvanceJob(jobState job.JobState, jobStage job.JobStage, ts time.Time, err error, db Database, notifs Notifs) (job.JobState, error) { jobState.Stage = jobStage - // Store how much time has elapsed since the stage last changed for this job - jobState.Params[job.JobParam_Elapsed] = time.Since(jobState.Ts).String() + if jobState.Params == nil { + jobState.Params = map[string]interface{}{} + } + // Store how much time the job spent during its previous stage. We only care about active jobs, i.e. those that have + // progressed beyond the "dequeued" stage. + if jobStage != job.JobStage_Dequeued { + jobState.Params[job.JobParam_WaitTime] = time.Since(jobState.Ts).String() + } jobState.Ts = ts if err != nil { jobState.Params[job.JobParam_Error] = err.Error()