From af1bf3ef0120f335af2e97cdd6a402df211df57c Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Wed, 8 Mar 2023 13:30:59 +1100 Subject: [PATCH 1/4] [v10] Make GHA pipeline structure more generic Make the GHA pipeline structure generic instead of assuming it only calls the linux-arm64 workflows. This backports part of dff5cd4042c3324a39de57c214fdbe2986f54edd from "Integrates distroless OCI publishing into drone (#22707)", but does not backport the distroless OCI pipelines themselves. --- .drone.yml | 27 ++++++++++----------------- dronegen/gha.go | 45 ++++++++++++++++++++++++++++----------------- dronegen/push.go | 15 ++++++++------- dronegen/tag.go | 13 ++++++++----- 4 files changed, 54 insertions(+), 46 deletions(-) diff --git a/.drone.yml b/.drone.yml index 9d96f1e4a6980..dde5e31bee8e7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1275,11 +1275,6 @@ image_pull_secrets: kind: pipeline type: kubernetes name: push-build-linux-arm64 -environment: - BUILDBOX_VERSION: teleport10 - GID: "1000" - RUNTIME: go1.19.8 - UID: "1000" trigger: event: include: @@ -1322,9 +1317,10 @@ steps: pull: if-not-exists commands: - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - - go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e -workflow - release-linux-arm64.yml -workflow-ref=${DRONE_BRANCH} -input oss-teleport-ref=${DRONE_COMMIT} - -input upload-artifacts=false -input oss-teleport-repo="${DRONE_REPO}" + - 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e + -tag-workflow -workflow release-linux-arm64.yml -workflow-ref=${DRONE_BRANCH} + -input oss-teleport-repo=${DRONE_REPO} -input oss-teleport-ref=${DRONE_COMMIT} + -input "upload-artifacts=false" ' environment: GHA_APP_KEY: from_secret: GITHUB_WORKFLOW_APP_PRIVATE_KEY @@ -5239,11 +5235,6 @@ image_pull_secrets: kind: pipeline type: kubernetes name: build-linux-arm64 -environment: - BUILDBOX_VERSION: teleport10 - GID: "1000" - RUNTIME: go1.19.8 - UID: "1000" trigger: event: include: @@ -5258,6 +5249,8 @@ workspace: path: /go clone: disable: true +depends_on: +- clean-up-previous-build steps: - name: Check out code image: docker:git @@ -5283,9 +5276,9 @@ steps: pull: if-not-exists commands: - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - - go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e -workflow - release-linux-arm64.yml -workflow-ref=${DRONE_TAG} -input oss-teleport-ref=${DRONE_TAG} - -input upload-artifacts=true -input oss-teleport-repo="${DRONE_REPO}" + - 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e + -tag-workflow -workflow release-linux-arm64.yml -workflow-ref=${DRONE_TAG} -input + oss-teleport-repo=${DRONE_REPO} -input oss-teleport-ref=${DRONE_TAG} -input "upload-artifacts=true" ' environment: GHA_APP_KEY: from_secret: GITHUB_WORKFLOW_APP_PRIVATE_KEY @@ -20247,6 +20240,6 @@ image_pull_secrets: - DOCKERHUB_CREDENTIALS --- kind: signature -hmac: c033b7e21ecf8503a8e5125e0e0d5b127dab1948f71c1f757788141e733045bd +hmac: 59835f9006a1945dcb345f0b740857d64b712982b83fd0ed45f5ada2a838775e ... diff --git a/dronegen/gha.go b/dronegen/gha.go index 8ffdfc07886df..1fb7526d56f6a 100644 --- a/dronegen/gha.go +++ b/dronegen/gha.go @@ -14,27 +14,42 @@ package main -import "fmt" +import ( + "fmt" + "strings" +) type ghaBuildType struct { buildType trigger - namePrefix string - uploadArtifacts bool - srcRefVar string - workflowRefVar string - slackOnError bool + pipelineName string + ghaWorkflow string + srcRefVar string + workflowRefVar string + slackOnError bool + dependsOn []string + inputs map[string]string } func ghaBuildPipeline(b ghaBuildType) pipeline { - p := newKubePipeline(fmt.Sprintf("%sbuild-%s-%s", b.namePrefix, b.os, b.arch)) + p := newKubePipeline(b.pipelineName) p.Trigger = b.trigger p.Workspace = workspace{Path: "/go"} - p.Environment = map[string]value{ - "BUILDBOX_VERSION": buildboxVersion, - "RUNTIME": goRuntime, - "UID": {raw: "1000"}, - "GID": {raw: "1000"}, + p.DependsOn = append(p.DependsOn, b.dependsOn...) + + var cmd strings.Builder + cmd.WriteString(`go run ./cmd/gh-trigger-workflow `) + cmd.WriteString(`-owner ${DRONE_REPO_OWNER} `) + cmd.WriteString(`-repo teleport.e `) + cmd.WriteString(`-tag-workflow `) + fmt.Fprintf(&cmd, `-workflow %s `, b.ghaWorkflow) + fmt.Fprintf(&cmd, `-workflow-ref=${%s} `, b.workflowRefVar) + + cmd.WriteString(`-input oss-teleport-repo=${DRONE_REPO} `) + fmt.Fprintf(&cmd, `-input oss-teleport-ref=${%s} `, b.srcRefVar) + + for k, v := range b.inputs { + fmt.Fprintf(&cmd, `-input "%s=%s" `, k, v) } p.Steps = []step{ @@ -56,11 +71,7 @@ func ghaBuildPipeline(b ghaBuildType) pipeline { }, Commands: []string{ `cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"`, - `go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e -workflow release-linux-arm64.yml ` + - fmt.Sprintf(`-workflow-ref=${%s} `, b.workflowRefVar) + - fmt.Sprintf(`-input oss-teleport-ref=${%s} `, b.srcRefVar) + - fmt.Sprintf(`-input upload-artifacts=%t `, b.uploadArtifacts) + - `-input oss-teleport-repo="${DRONE_REPO}"`, + cmd.String(), }, }, } diff --git a/dronegen/push.go b/dronegen/push.go index 30206b02b8861..2b5184c672c78 100644 --- a/dronegen/push.go +++ b/dronegen/push.go @@ -73,13 +73,14 @@ func pushPipelines() []pipeline { } ps = append(ps, ghaBuildPipeline(ghaBuildType{ - buildType: buildType{os: "linux", arch: "arm64"}, - trigger: triggerPush, - namePrefix: "push-", - uploadArtifacts: false, - slackOnError: true, - srcRefVar: "DRONE_COMMIT", - workflowRefVar: "DRONE_BRANCH", + buildType: buildType{os: "linux", arch: "arm64"}, + trigger: triggerPush, + pipelineName: "push-build-linux-arm64", + ghaWorkflow: "release-linux-arm64.yml", + slackOnError: true, + srcRefVar: "DRONE_COMMIT", + workflowRefVar: "DRONE_BRANCH", + inputs: map[string]string{"upload-artifacts": "false"}, })) // Only amd64 Windows is supported for now. diff --git a/dronegen/tag.go b/dronegen/tag.go index edb22b6ff2913..619e6d8616104 100644 --- a/dronegen/tag.go +++ b/dronegen/tag.go @@ -191,11 +191,14 @@ func tagPipelines() []pipeline { } ps = append(ps, ghaBuildPipeline(ghaBuildType{ - buildType: buildType{os: "linux", arch: "arm64", fips: false}, - trigger: triggerTag, - uploadArtifacts: true, - srcRefVar: "DRONE_TAG", - workflowRefVar: "DRONE_TAG", + buildType: buildType{os: "linux", arch: "arm64", fips: false}, + trigger: triggerTag, + pipelineName: "build-linux-arm64", + ghaWorkflow: "release-linux-arm64.yml", + srcRefVar: "DRONE_TAG", + workflowRefVar: "DRONE_TAG", + dependsOn: []string{tagCleanupPipelineName}, + inputs: map[string]string{"upload-artifacts": "true"}, })) // Only amd64 Windows is supported for now. From b6826b092d83cf13a8f19d87f611c6d08b67e7b3 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Tue, 14 Mar 2023 12:45:23 +1100 Subject: [PATCH 2/4] [v10] Removes unnecessary workflow arguments for GHA pipelines Removes unnecessary workflow arguments from the Drone workflow invocation. Refactors some of the `dronegen` generators to make this easier to express insife `dronegen` This backports "Fix OCI promotion (#22867)" without the changes to the OCI pipelines as they have not been backported to v11. --- dronegen/gha.go | 23 +++++++++++++---------- dronegen/push.go | 16 ++++++++-------- dronegen/tag.go | 16 ++++++++-------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/dronegen/gha.go b/dronegen/gha.go index 1fb7526d56f6a..6d0c770c67d4c 100644 --- a/dronegen/gha.go +++ b/dronegen/gha.go @@ -22,13 +22,13 @@ import ( type ghaBuildType struct { buildType trigger - pipelineName string - ghaWorkflow string - srcRefVar string - workflowRefVar string - slackOnError bool - dependsOn []string - inputs map[string]string + pipelineName string + ghaWorkflow string + srcRefVar string + workflowRef string + slackOnError bool + dependsOn []string + inputs map[string]string } func ghaBuildPipeline(b ghaBuildType) pipeline { @@ -43,10 +43,13 @@ func ghaBuildPipeline(b ghaBuildType) pipeline { cmd.WriteString(`-repo teleport.e `) cmd.WriteString(`-tag-workflow `) fmt.Fprintf(&cmd, `-workflow %s `, b.ghaWorkflow) - fmt.Fprintf(&cmd, `-workflow-ref=${%s} `, b.workflowRefVar) + fmt.Fprintf(&cmd, `-workflow-ref=%s `, b.workflowRef) - cmd.WriteString(`-input oss-teleport-repo=${DRONE_REPO} `) - fmt.Fprintf(&cmd, `-input oss-teleport-ref=${%s} `, b.srcRefVar) + // If we don't need to build teleport... + if b.srcRefVar != "" { + cmd.WriteString(`-input oss-teleport-repo=${DRONE_REPO} `) + fmt.Fprintf(&cmd, `-input oss-teleport-ref=${%s} `, b.srcRefVar) + } for k, v := range b.inputs { fmt.Fprintf(&cmd, `-input "%s=%s" `, k, v) diff --git a/dronegen/push.go b/dronegen/push.go index 2b5184c672c78..216fe5b8d020a 100644 --- a/dronegen/push.go +++ b/dronegen/push.go @@ -73,14 +73,14 @@ func pushPipelines() []pipeline { } ps = append(ps, ghaBuildPipeline(ghaBuildType{ - buildType: buildType{os: "linux", arch: "arm64"}, - trigger: triggerPush, - pipelineName: "push-build-linux-arm64", - ghaWorkflow: "release-linux-arm64.yml", - slackOnError: true, - srcRefVar: "DRONE_COMMIT", - workflowRefVar: "DRONE_BRANCH", - inputs: map[string]string{"upload-artifacts": "false"}, + buildType: buildType{os: "linux", arch: "arm64"}, + trigger: triggerPush, + pipelineName: "push-build-linux-arm64", + ghaWorkflow: "release-linux-arm64.yml", + slackOnError: true, + srcRefVar: "DRONE_COMMIT", + workflowRef: "${DRONE_BRANCH}", + inputs: map[string]string{"upload-artifacts": "false"}, })) // Only amd64 Windows is supported for now. diff --git a/dronegen/tag.go b/dronegen/tag.go index 619e6d8616104..bdbeadf8cd716 100644 --- a/dronegen/tag.go +++ b/dronegen/tag.go @@ -191,14 +191,14 @@ func tagPipelines() []pipeline { } ps = append(ps, ghaBuildPipeline(ghaBuildType{ - buildType: buildType{os: "linux", arch: "arm64", fips: false}, - trigger: triggerTag, - pipelineName: "build-linux-arm64", - ghaWorkflow: "release-linux-arm64.yml", - srcRefVar: "DRONE_TAG", - workflowRefVar: "DRONE_TAG", - dependsOn: []string{tagCleanupPipelineName}, - inputs: map[string]string{"upload-artifacts": "true"}, + buildType: buildType{os: "linux", arch: "arm64", fips: false}, + trigger: triggerTag, + pipelineName: "build-linux-arm64", + ghaWorkflow: "release-linux-arm64.yml", + srcRefVar: "DRONE_TAG", + workflowRef: "${DRONE_TAG}", + dependsOn: []string{tagCleanupPipelineName}, + inputs: map[string]string{"upload-artifacts": "true"}, })) // Only amd64 Windows is supported for now. From f6f658cfc1e20ffe0906f86e1be134feab0f440b Mon Sep 17 00:00:00 2001 From: Jakub Nyckowski Date: Mon, 20 Mar 2023 19:23:17 -0400 Subject: [PATCH 3/4] [v10] Increase GHA build timeout Our ARM64 build fails quite frequently due to 30 minutes timeout. This PR increases the build timeout to one hour. --- .drone.yml | 9 +++++---- dronegen/gha.go | 3 +++ dronegen/push.go | 6 +++++- dronegen/tag.go | 2 ++ 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.drone.yml b/.drone.yml index dde5e31bee8e7..ed97bbca9afba 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1318,7 +1318,7 @@ steps: commands: - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e - -tag-workflow -workflow release-linux-arm64.yml -workflow-ref=${DRONE_BRANCH} + -tag-workflow -timeout 1h0m0s -workflow release-linux-arm64.yml -workflow-ref=${DRONE_BRANCH} -input oss-teleport-repo=${DRONE_REPO} -input oss-teleport-ref=${DRONE_COMMIT} -input "upload-artifacts=false" ' environment: @@ -5277,8 +5277,9 @@ steps: commands: - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e - -tag-workflow -workflow release-linux-arm64.yml -workflow-ref=${DRONE_TAG} -input - oss-teleport-repo=${DRONE_REPO} -input oss-teleport-ref=${DRONE_TAG} -input "upload-artifacts=true" ' + -tag-workflow -timeout 1h0m0s -workflow release-linux-arm64.yml -workflow-ref=${DRONE_TAG} + -input oss-teleport-repo=${DRONE_REPO} -input oss-teleport-ref=${DRONE_TAG} -input + "upload-artifacts=true" ' environment: GHA_APP_KEY: from_secret: GITHUB_WORKFLOW_APP_PRIVATE_KEY @@ -20240,6 +20241,6 @@ image_pull_secrets: - DOCKERHUB_CREDENTIALS --- kind: signature -hmac: 59835f9006a1945dcb345f0b740857d64b712982b83fd0ed45f5ada2a838775e +hmac: 0f66eef0fd310ee80bb80671bffba2bb97b2f426888260d6bec8fb5aedd5c860 ... diff --git a/dronegen/gha.go b/dronegen/gha.go index 6d0c770c67d4c..9ab3dcbb6b58c 100644 --- a/dronegen/gha.go +++ b/dronegen/gha.go @@ -17,6 +17,7 @@ package main import ( "fmt" "strings" + "time" ) type ghaBuildType struct { @@ -26,6 +27,7 @@ type ghaBuildType struct { ghaWorkflow string srcRefVar string workflowRef string + timeout time.Duration slackOnError bool dependsOn []string inputs map[string]string @@ -42,6 +44,7 @@ func ghaBuildPipeline(b ghaBuildType) pipeline { cmd.WriteString(`-owner ${DRONE_REPO_OWNER} `) cmd.WriteString(`-repo teleport.e `) cmd.WriteString(`-tag-workflow `) + fmt.Fprintf(&cmd, `-timeout %s `, b.timeout.String()) fmt.Fprintf(&cmd, `-workflow %s `, b.ghaWorkflow) fmt.Fprintf(&cmd, `-workflow-ref=%s `, b.workflowRef) diff --git a/dronegen/push.go b/dronegen/push.go index 216fe5b8d020a..e3d5a5502d61c 100644 --- a/dronegen/push.go +++ b/dronegen/push.go @@ -14,7 +14,10 @@ package main -import "fmt" +import ( + "fmt" + "time" +) // pushCheckoutCommands builds a list of commands for Drone to check out a git commit on a push build func pushCheckoutCommands(b buildType) []string { @@ -77,6 +80,7 @@ func pushPipelines() []pipeline { trigger: triggerPush, pipelineName: "push-build-linux-arm64", ghaWorkflow: "release-linux-arm64.yml", + timeout: 60 * time.Minute, slackOnError: true, srcRefVar: "DRONE_COMMIT", workflowRef: "${DRONE_BRANCH}", diff --git a/dronegen/tag.go b/dronegen/tag.go index bdbeadf8cd716..5c3c0b8509770 100644 --- a/dronegen/tag.go +++ b/dronegen/tag.go @@ -17,6 +17,7 @@ package main import ( "fmt" "strings" + "time" ) const ( @@ -197,6 +198,7 @@ func tagPipelines() []pipeline { ghaWorkflow: "release-linux-arm64.yml", srcRefVar: "DRONE_TAG", workflowRef: "${DRONE_TAG}", + timeout: 60 * time.Minute, dependsOn: []string{tagCleanupPipelineName}, inputs: map[string]string{"upload-artifacts": "true"}, })) From 9fa43e6f2d05749d13afdaad10b23242c1da9b75 Mon Sep 17 00:00:00 2001 From: Cam Hutchison Date: Thu, 6 Apr 2023 07:27:53 +1000 Subject: [PATCH 4/4] [v10] dronegen: Sort workflow inputs for stable output Sort the GitHub Actions inputs when generating the `gh-trigger-workflow` command line so that it does not randomly change order, as happens when iterating a map directly. This is a backport of a single commit of #24102 (drone: Switch Mac (darwin) pipelines to GitHub Actions), as it changes dronegen, but the pipelines themselves are not being backported to `branch/v10`. --- dronegen/gha.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/dronegen/gha.go b/dronegen/gha.go index 9ab3dcbb6b58c..af9082d897e5f 100644 --- a/dronegen/gha.go +++ b/dronegen/gha.go @@ -16,8 +16,11 @@ package main import ( "fmt" + "sort" "strings" "time" + + "golang.org/x/exp/maps" ) type ghaBuildType struct { @@ -54,8 +57,12 @@ func ghaBuildPipeline(b ghaBuildType) pipeline { fmt.Fprintf(&cmd, `-input oss-teleport-ref=${%s} `, b.srcRefVar) } - for k, v := range b.inputs { - fmt.Fprintf(&cmd, `-input "%s=%s" `, k, v) + // Sort inputs so the are output in a consistent order to avoid + // spurious changes in the generated drone config. + keys := maps.Keys(b.inputs) + sort.Strings(keys) + for _, k := range keys { + fmt.Fprintf(&cmd, `-input "%s=%s" `, k, b.inputs[k]) } p.Steps = []step{