From 36b33e15d7dd7a84dbb7785454e5ba0230ad6c93 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Wed, 8 Mar 2023 13:30:59 +1100 Subject: [PATCH 1/4] [v11] Make GHA pipeline structure more generic Make the GHA pipeline structure generic instead of assuming it only calls the linux-arm64 workflows. This backports part of dff5cd4042c3324a39de57c214fdbe2986f54edd from "Integrates distroless OCI publishing into drone (#22707)", but does not backport the distroless OCI pipelines themselves. --- .drone.yml | 27 ++++++++++----------------- dronegen/gha.go | 45 ++++++++++++++++++++++++++++----------------- dronegen/push.go | 15 ++++++++------- dronegen/tag.go | 13 ++++++++----- 4 files changed, 54 insertions(+), 46 deletions(-) diff --git a/.drone.yml b/.drone.yml index 5f65008ffdca3..7df92fd3edcd3 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1319,11 +1319,6 @@ image_pull_secrets: kind: pipeline type: kubernetes name: push-build-linux-arm64 -environment: - BUILDBOX_VERSION: teleport11 - GID: "1000" - RUNTIME: go1.20.3 - UID: "1000" trigger: event: include: @@ -1366,9 +1361,10 @@ steps: pull: if-not-exists commands: - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - - go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e -workflow - release-linux-arm64.yml -workflow-ref=${DRONE_BRANCH} -input oss-teleport-ref=${DRONE_COMMIT} - -input upload-artifacts=false -input oss-teleport-repo="${DRONE_REPO}" + - 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e + -tag-workflow -workflow release-linux-arm64.yml -workflow-ref=${DRONE_BRANCH} + -input oss-teleport-repo=${DRONE_REPO} -input oss-teleport-ref=${DRONE_COMMIT} + -input "upload-artifacts=false" ' environment: GHA_APP_KEY: from_secret: GITHUB_WORKFLOW_APP_PRIVATE_KEY @@ -5284,11 +5280,6 @@ image_pull_secrets: kind: pipeline type: kubernetes name: build-linux-arm64 -environment: - BUILDBOX_VERSION: teleport11 - GID: "1000" - RUNTIME: go1.20.3 - UID: "1000" trigger: event: include: @@ -5303,6 +5294,8 @@ workspace: path: /go clone: disable: true +depends_on: +- clean-up-previous-build steps: - name: Check out code image: docker:git @@ -5328,9 +5321,9 @@ steps: pull: if-not-exists commands: - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - - go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e -workflow - release-linux-arm64.yml -workflow-ref=${DRONE_TAG} -input oss-teleport-ref=${DRONE_TAG} - -input upload-artifacts=true -input oss-teleport-repo="${DRONE_REPO}" + - 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e + -tag-workflow -workflow release-linux-arm64.yml -workflow-ref=${DRONE_TAG} -input + oss-teleport-repo=${DRONE_REPO} -input oss-teleport-ref=${DRONE_TAG} -input "upload-artifacts=true" ' environment: GHA_APP_KEY: from_secret: GITHUB_WORKFLOW_APP_PRIVATE_KEY @@ -20291,6 +20284,6 @@ image_pull_secrets: - DOCKERHUB_CREDENTIALS --- kind: signature -hmac: 5f920d6511d5e89e2bfa788d11b42a8db5b3f6f5c98500b137b6fba4d1ab03ee +hmac: 045a5ebf373b9471ba92e45571e9f749550a20b5cd11520b89fc3ce5266e9e7b ... diff --git a/dronegen/gha.go b/dronegen/gha.go index 8ffdfc07886df..1fb7526d56f6a 100644 --- a/dronegen/gha.go +++ b/dronegen/gha.go @@ -14,27 +14,42 @@ package main -import "fmt" +import ( + "fmt" + "strings" +) type ghaBuildType struct { buildType trigger - namePrefix string - uploadArtifacts bool - srcRefVar string - workflowRefVar string - slackOnError bool + pipelineName string + ghaWorkflow string + srcRefVar string + workflowRefVar string + slackOnError bool + dependsOn []string + inputs map[string]string } func ghaBuildPipeline(b ghaBuildType) pipeline { - p := newKubePipeline(fmt.Sprintf("%sbuild-%s-%s", b.namePrefix, b.os, b.arch)) + p := newKubePipeline(b.pipelineName) p.Trigger = b.trigger p.Workspace = workspace{Path: "/go"} - p.Environment = map[string]value{ - "BUILDBOX_VERSION": buildboxVersion, - "RUNTIME": goRuntime, - "UID": {raw: "1000"}, - "GID": {raw: "1000"}, + p.DependsOn = append(p.DependsOn, b.dependsOn...) + + var cmd strings.Builder + cmd.WriteString(`go run ./cmd/gh-trigger-workflow `) + cmd.WriteString(`-owner ${DRONE_REPO_OWNER} `) + cmd.WriteString(`-repo teleport.e `) + cmd.WriteString(`-tag-workflow `) + fmt.Fprintf(&cmd, `-workflow %s `, b.ghaWorkflow) + fmt.Fprintf(&cmd, `-workflow-ref=${%s} `, b.workflowRefVar) + + cmd.WriteString(`-input oss-teleport-repo=${DRONE_REPO} `) + fmt.Fprintf(&cmd, `-input oss-teleport-ref=${%s} `, b.srcRefVar) + + for k, v := range b.inputs { + fmt.Fprintf(&cmd, `-input "%s=%s" `, k, v) } p.Steps = []step{ @@ -56,11 +71,7 @@ func ghaBuildPipeline(b ghaBuildType) pipeline { }, Commands: []string{ `cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"`, - `go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e -workflow release-linux-arm64.yml ` + - fmt.Sprintf(`-workflow-ref=${%s} `, b.workflowRefVar) + - fmt.Sprintf(`-input oss-teleport-ref=${%s} `, b.srcRefVar) + - fmt.Sprintf(`-input upload-artifacts=%t `, b.uploadArtifacts) + - `-input oss-teleport-repo="${DRONE_REPO}"`, + cmd.String(), }, }, } diff --git a/dronegen/push.go b/dronegen/push.go index 80802c2e15961..888c2c32adf67 100644 --- a/dronegen/push.go +++ b/dronegen/push.go @@ -72,13 +72,14 @@ func pushPipelines() []pipeline { } ps = append(ps, ghaBuildPipeline(ghaBuildType{ - buildType: buildType{os: "linux", arch: "arm64"}, - trigger: triggerPush, - namePrefix: "push-", - uploadArtifacts: false, - slackOnError: true, - srcRefVar: "DRONE_COMMIT", - workflowRefVar: "DRONE_BRANCH", + buildType: buildType{os: "linux", arch: "arm64"}, + trigger: triggerPush, + pipelineName: "push-build-linux-arm64", + ghaWorkflow: "release-linux-arm64.yml", + slackOnError: true, + srcRefVar: "DRONE_COMMIT", + workflowRefVar: "DRONE_BRANCH", + inputs: map[string]string{"upload-artifacts": "false"}, })) // Only amd64 Windows is supported for now. diff --git a/dronegen/tag.go b/dronegen/tag.go index 3b3698507ca7f..275bb251b86ba 100644 --- a/dronegen/tag.go +++ b/dronegen/tag.go @@ -189,11 +189,14 @@ func tagPipelines() []pipeline { } ps = append(ps, ghaBuildPipeline(ghaBuildType{ - buildType: buildType{os: "linux", arch: "arm64", fips: false}, - trigger: triggerTag, - uploadArtifacts: true, - srcRefVar: "DRONE_TAG", - workflowRefVar: "DRONE_TAG", + buildType: buildType{os: "linux", arch: "arm64", fips: false}, + trigger: triggerTag, + pipelineName: "build-linux-arm64", + ghaWorkflow: "release-linux-arm64.yml", + srcRefVar: "DRONE_TAG", + workflowRefVar: "DRONE_TAG", + dependsOn: []string{tagCleanupPipelineName}, + inputs: map[string]string{"upload-artifacts": "true"}, })) // Only amd64 Windows is supported for now. From 0be86d30434023cc4124ec82504295692c01152c Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Tue, 14 Mar 2023 12:45:23 +1100 Subject: [PATCH 2/4] [v11] Removes unnecessary workflow arguments for GHA pipelines Removes unnecessary workflow arguments from the Drone workflow invocation. Refactors some of the `dronegen` generators to make this easier to express insife `dronegen` This backports "Fix OCI promotion (#22867)" without the changes to the OCI pipelines as they have not been backported to v11. --- dronegen/gha.go | 23 +++++++++++++---------- dronegen/push.go | 16 ++++++++-------- dronegen/tag.go | 16 ++++++++-------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/dronegen/gha.go b/dronegen/gha.go index 1fb7526d56f6a..6d0c770c67d4c 100644 --- a/dronegen/gha.go +++ b/dronegen/gha.go @@ -22,13 +22,13 @@ import ( type ghaBuildType struct { buildType trigger - pipelineName string - ghaWorkflow string - srcRefVar string - workflowRefVar string - slackOnError bool - dependsOn []string - inputs map[string]string + pipelineName string + ghaWorkflow string + srcRefVar string + workflowRef string + slackOnError bool + dependsOn []string + inputs map[string]string } func ghaBuildPipeline(b ghaBuildType) pipeline { @@ -43,10 +43,13 @@ func ghaBuildPipeline(b ghaBuildType) pipeline { cmd.WriteString(`-repo teleport.e `) cmd.WriteString(`-tag-workflow `) fmt.Fprintf(&cmd, `-workflow %s `, b.ghaWorkflow) - fmt.Fprintf(&cmd, `-workflow-ref=${%s} `, b.workflowRefVar) + fmt.Fprintf(&cmd, `-workflow-ref=%s `, b.workflowRef) - cmd.WriteString(`-input oss-teleport-repo=${DRONE_REPO} `) - fmt.Fprintf(&cmd, `-input oss-teleport-ref=${%s} `, b.srcRefVar) + // If we don't need to build teleport... + if b.srcRefVar != "" { + cmd.WriteString(`-input oss-teleport-repo=${DRONE_REPO} `) + fmt.Fprintf(&cmd, `-input oss-teleport-ref=${%s} `, b.srcRefVar) + } for k, v := range b.inputs { fmt.Fprintf(&cmd, `-input "%s=%s" `, k, v) diff --git a/dronegen/push.go b/dronegen/push.go index 888c2c32adf67..d35cf9da45a1f 100644 --- a/dronegen/push.go +++ b/dronegen/push.go @@ -72,14 +72,14 @@ func pushPipelines() []pipeline { } ps = append(ps, ghaBuildPipeline(ghaBuildType{ - buildType: buildType{os: "linux", arch: "arm64"}, - trigger: triggerPush, - pipelineName: "push-build-linux-arm64", - ghaWorkflow: "release-linux-arm64.yml", - slackOnError: true, - srcRefVar: "DRONE_COMMIT", - workflowRefVar: "DRONE_BRANCH", - inputs: map[string]string{"upload-artifacts": "false"}, + buildType: buildType{os: "linux", arch: "arm64"}, + trigger: triggerPush, + pipelineName: "push-build-linux-arm64", + ghaWorkflow: "release-linux-arm64.yml", + slackOnError: true, + srcRefVar: "DRONE_COMMIT", + workflowRef: "${DRONE_BRANCH}", + inputs: map[string]string{"upload-artifacts": "false"}, })) // Only amd64 Windows is supported for now. diff --git a/dronegen/tag.go b/dronegen/tag.go index 275bb251b86ba..55051800a44a6 100644 --- a/dronegen/tag.go +++ b/dronegen/tag.go @@ -189,14 +189,14 @@ func tagPipelines() []pipeline { } ps = append(ps, ghaBuildPipeline(ghaBuildType{ - buildType: buildType{os: "linux", arch: "arm64", fips: false}, - trigger: triggerTag, - pipelineName: "build-linux-arm64", - ghaWorkflow: "release-linux-arm64.yml", - srcRefVar: "DRONE_TAG", - workflowRefVar: "DRONE_TAG", - dependsOn: []string{tagCleanupPipelineName}, - inputs: map[string]string{"upload-artifacts": "true"}, + buildType: buildType{os: "linux", arch: "arm64", fips: false}, + trigger: triggerTag, + pipelineName: "build-linux-arm64", + ghaWorkflow: "release-linux-arm64.yml", + srcRefVar: "DRONE_TAG", + workflowRef: "${DRONE_TAG}", + dependsOn: []string{tagCleanupPipelineName}, + inputs: map[string]string{"upload-artifacts": "true"}, })) // Only amd64 Windows is supported for now. From 5c1b3813076d257059fb5213bc39c2c122357a12 Mon Sep 17 00:00:00 2001 From: Jakub Nyckowski Date: Mon, 20 Mar 2023 19:23:17 -0400 Subject: [PATCH 3/4] [v11] Increase GHA build timeout Our ARM64 build fails quite frequently due to 30 minutes timeout. This PR increases the build timeout to one hour. --- dronegen/gha.go | 3 +++ dronegen/push.go | 6 +++++- dronegen/tag.go | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dronegen/gha.go b/dronegen/gha.go index 6d0c770c67d4c..9ab3dcbb6b58c 100644 --- a/dronegen/gha.go +++ b/dronegen/gha.go @@ -17,6 +17,7 @@ package main import ( "fmt" "strings" + "time" ) type ghaBuildType struct { @@ -26,6 +27,7 @@ type ghaBuildType struct { ghaWorkflow string srcRefVar string workflowRef string + timeout time.Duration slackOnError bool dependsOn []string inputs map[string]string @@ -42,6 +44,7 @@ func ghaBuildPipeline(b ghaBuildType) pipeline { cmd.WriteString(`-owner ${DRONE_REPO_OWNER} `) cmd.WriteString(`-repo teleport.e `) cmd.WriteString(`-tag-workflow `) + fmt.Fprintf(&cmd, `-timeout %s `, b.timeout.String()) fmt.Fprintf(&cmd, `-workflow %s `, b.ghaWorkflow) fmt.Fprintf(&cmd, `-workflow-ref=%s `, b.workflowRef) diff --git a/dronegen/push.go b/dronegen/push.go index d35cf9da45a1f..1cea3b2ec4182 100644 --- a/dronegen/push.go +++ b/dronegen/push.go @@ -14,7 +14,10 @@ package main -import "fmt" +import ( + "fmt" + "time" +) // pushCheckoutCommands builds a list of commands for Drone to check out a git commit on a push build func pushCheckoutCommands(b buildType) []string { @@ -76,6 +79,7 @@ func pushPipelines() []pipeline { trigger: triggerPush, pipelineName: "push-build-linux-arm64", ghaWorkflow: "release-linux-arm64.yml", + timeout: 60 * time.Minute, slackOnError: true, srcRefVar: "DRONE_COMMIT", workflowRef: "${DRONE_BRANCH}", diff --git a/dronegen/tag.go b/dronegen/tag.go index 55051800a44a6..e470fe171cb65 100644 --- a/dronegen/tag.go +++ b/dronegen/tag.go @@ -17,6 +17,7 @@ package main import ( "fmt" "strings" + "time" ) const ( @@ -195,6 +196,7 @@ func tagPipelines() []pipeline { ghaWorkflow: "release-linux-arm64.yml", srcRefVar: "DRONE_TAG", workflowRef: "${DRONE_TAG}", + timeout: 60 * time.Minute, dependsOn: []string{tagCleanupPipelineName}, inputs: map[string]string{"upload-artifacts": "true"}, })) From ac7935c31d32bb9e3827bf809fe3efc693c839d9 Mon Sep 17 00:00:00 2001 From: Cam Hutchison Date: Thu, 20 Apr 2023 06:10:57 +1000 Subject: [PATCH 4/4] Run dronegen for timeout params --- .drone.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.drone.yml b/.drone.yml index 7df92fd3edcd3..a065f6092996c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1362,7 +1362,7 @@ steps: commands: - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e - -tag-workflow -workflow release-linux-arm64.yml -workflow-ref=${DRONE_BRANCH} + -tag-workflow -timeout 1h0m0s -workflow release-linux-arm64.yml -workflow-ref=${DRONE_BRANCH} -input oss-teleport-repo=${DRONE_REPO} -input oss-teleport-ref=${DRONE_COMMIT} -input "upload-artifacts=false" ' environment: @@ -5322,8 +5322,9 @@ steps: commands: - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e - -tag-workflow -workflow release-linux-arm64.yml -workflow-ref=${DRONE_TAG} -input - oss-teleport-repo=${DRONE_REPO} -input oss-teleport-ref=${DRONE_TAG} -input "upload-artifacts=true" ' + -tag-workflow -timeout 1h0m0s -workflow release-linux-arm64.yml -workflow-ref=${DRONE_TAG} + -input oss-teleport-repo=${DRONE_REPO} -input oss-teleport-ref=${DRONE_TAG} -input + "upload-artifacts=true" ' environment: GHA_APP_KEY: from_secret: GITHUB_WORKFLOW_APP_PRIVATE_KEY @@ -20284,6 +20285,6 @@ image_pull_secrets: - DOCKERHUB_CREDENTIALS --- kind: signature -hmac: 045a5ebf373b9471ba92e45571e9f749550a20b5cd11520b89fc3ce5266e9e7b +hmac: b797d960837073d77869974db84306e894dc81b4a8cde6e24fdb7056eb15a65e ...