diff --git a/.drone.yml b/.drone.yml index dfce6c2df4c18..318fc4c4a1df6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -481,10 +481,8 @@ steps: - echo "$RELEASES_CERT" | base64 -d > "$RELCLI_CERT" - echo "$RELEASES_KEY" | base64 -d > "$RELCLI_KEY" - trap "rm -rf /tmpfs/creds" EXIT - - |- - docker run -i -v /tmpfs/creds:/tmpfs/creds \ - -e DRONE_REPO -e DRONE_TAG -e RELCLI_BASE_URL -e RELCLI_CERT -e RELCLI_KEY \ - $RELCLI_IMAGE auto_destroy -f -v 6 + - docker run -i -v /tmpfs/creds:/tmpfs/creds -e DRONE_REPO -e DRONE_TAG -e RELCLI_BASE_URL + -e RELCLI_CERT -e RELCLI_KEY $RELCLI_IMAGE auto_destroy -f -v 6 environment: RELCLI_BASE_URL: https://releases-prod.platform.teleport.sh RELCLI_CERT: /tmpfs/creds/releases.crt @@ -19227,10 +19225,9 @@ steps: - echo "$RELEASES_CERT" | base64 -d > "$RELCLI_CERT" - echo "$RELEASES_KEY" | base64 -d > "$RELCLI_KEY" - trap "rm -rf /tmpfs/creds" EXIT - - |- - docker run -i -v /tmpfs/creds:/tmpfs/creds \ - -e DRONE_REPO -e DRONE_TAG -e RELCLI_BASE_URL -e RELCLI_CERT -e RELCLI_KEY \ - $RELCLI_IMAGE auto_publish -f -v 6 + - for i in $(seq 10); do docker run -i -v /tmpfs/creds:/tmpfs/creds -e DRONE_REPO + -e DRONE_TAG -e RELCLI_BASE_URL -e RELCLI_CERT -e RELCLI_KEY $RELCLI_IMAGE auto_publish + -f -v 6 && break; done || false environment: RELCLI_BASE_URL: https://releases-prod.platform.teleport.sh RELCLI_CERT: /tmpfs/creds/releases.crt @@ -19269,6 +19266,6 @@ image_pull_secrets: - DOCKERHUB_CREDENTIALS --- kind: signature -hmac: 4fd196b5cbb42ac731a5cb7afc254d3e935e8995b7500cc45f69456235cbe1a0 +hmac: 07b6aca0cc517b436f02bf9a850bcc9cdab6ec56668441b0313b7b180a335627 ... diff --git a/dronegen/relcli.go b/dronegen/relcli.go index 8b580c06a6b5d..d073dabda2ee3 100644 --- a/dronegen/relcli.go +++ b/dronegen/relcli.go @@ -14,6 +14,10 @@ package main +import ( + "strings" +) + const relcliImage = "146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/relcli:master-57a5d42-20230412T1204687" func relcliPipeline(trigger trigger, name string, stepName string, command string) pipeline { @@ -66,6 +70,25 @@ func pullRelcliStep(awsConfigVolumeRef volumeRef) step { } func executeRelcliStep(name string, command string) step { + commands := []string{ + `mkdir -p /tmpfs/creds`, + `echo "$RELEASES_CERT" | base64 -d > "$RELCLI_CERT"`, + `echo "$RELEASES_KEY" | base64 -d > "$RELCLI_KEY"`, + `trap "rm -rf /tmpfs/creds" EXIT`, + } + + runReleaseServerCLICommand := "docker run -i -v /tmpfs/creds:/tmpfs/creds " + + "-e DRONE_REPO -e DRONE_TAG -e RELCLI_BASE_URL -e RELCLI_CERT -e RELCLI_KEY " + + "$RELCLI_IMAGE " + command + + // This is a workaround for a release server issue, and should be removed after the issue is fixed. + // The release server publish step does not fail on or after the third step, consistently. + if strings.HasPrefix(command, "auto_publish") { + // Retry the command up to 10 times until success, and fail if none succeed. + runReleaseServerCLICommand = `for i in $(seq 10); do ` + runReleaseServerCLICommand + ` && break; done || false` + } + commands = append(commands, runReleaseServerCLICommand) + return step{ Name: name, Image: "docker:git", @@ -76,15 +99,7 @@ func executeRelcliStep(name string, command string) step { "RELCLI_CERT": {raw: "/tmpfs/creds/releases.crt"}, "RELCLI_KEY": {raw: "/tmpfs/creds/releases.key"}, }, - Volumes: []volumeRef{volumeRefDocker, volumeRefTmpfs, volumeRefAwsConfig}, - Commands: []string{ - `mkdir -p /tmpfs/creds`, - `echo "$RELEASES_CERT" | base64 -d > "$RELCLI_CERT"`, - `echo "$RELEASES_KEY" | base64 -d > "$RELCLI_KEY"`, - `trap "rm -rf /tmpfs/creds" EXIT`, - `docker run -i -v /tmpfs/creds:/tmpfs/creds \ - -e DRONE_REPO -e DRONE_TAG -e RELCLI_BASE_URL -e RELCLI_CERT -e RELCLI_KEY \ - $RELCLI_IMAGE ` + command, - }, + Volumes: []volumeRef{volumeRefDocker, volumeRefTmpfs, volumeRefAwsConfig}, + Commands: commands, } }