ci: Daily Releasable Artifacts pipeline

.ci/packaging.groovy

@@ -0,0 +1,157 @@
+#!/usr/bin/env groovy
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    REPO = 'apm-server'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
+    SLACK_CHANNEL = '#apm-server'
+    NOTIFY_TO = 'build-apm+apm-server@elastic.co'
+    JOB_GCS_BUCKET = credentials('gcs-bucket')
+    JOB_GCS_CREDENTIALS = 'apm-ci-gcs-plugin'
+    SNAPSHOT = "true"
+    DOCKER_SECRET = 'secret/apm-team/ci/docker-registry/prod'
+    DOCKER_REGISTRY = 'docker.elastic.co'
+  }
+  options {
+    timeout(time: 2, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '100', artifactNumToKeepStr: '30', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
+    quietPeriod(10)
+  }
+  triggers {
+    // disable upstream trigger on a PR basis
+    upstream("apm-server/apm-server-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
+  }
+  stages {
+    stage('Filter build') {
+      agent { label 'ubuntu-18 && immutable' }
+      when {
+        beforeAgent true
+        anyOf {
+          triggeredBy cause: "IssueCommentCause"
+          expression {
+            def ret = isUserTrigger() || isUpstreamTrigger()
+            if(!ret){
+              currentBuild.result = 'NOT_BUILT'
+              currentBuild.description = "The build has been skipped"
+              currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
+              echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
+            }
+            return ret
+          }
+        }
+      }
+      environment {
+        PATH = "${env.PATH}:${env.WORKSPACE}/bin"
+        HOME = "${env.WORKSPACE}"
+      }
+      stages {
+        stage('Checkout') {
+          options { skipDefaultCheckout() }
+          steps {
+            pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ])
+            deleteDir()
+            gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: false,
+                        shallow: false, reference: "/var/lib/jenkins/.git-references/${REPO}.git")
+            stash allowEmpty: true, name: 'source', useDefaultExcludes: false
+            // set environment variables globally since they are used afterwards but GIT_BASE_COMMIT won't
+            // be available until gitCheckout is executed.
+            setEnvVar('URI_SUFFIX', "commits/${env.GIT_BASE_COMMIT}")
+            // JOB_GCS_BUCKET contains the bucket and some folders, let's build the folder structure
+            setEnvVar('PATH_PREFIX', "${JOB_GCS_BUCKET.contains('/') ? JOB_GCS_BUCKET.substring(JOB_GCS_BUCKET.indexOf('/') + 1) + '/' + env.URI_SUFFIX : env.URI_SUFFIX}")
+          }
+        }
+        stage('Package') {
+          options { skipDefaultCheckout() }
+          matrix {
+            agent {
+              label "${PLATFORM}"
+            }
+            axes {
+              axis {
+                name 'PLATFORM'
+                values 'linux && immutable', 'arm'
+              }
+            }
+            stages {
+              stage('Package') {
+                environment {
+                  PLATFORMS = "${isArm() ? 'linux/arm64' : ''}"
+                  PACKAGES = "${isArm() ? 'docker' : ''}"
+                }
+                steps {
+                  deleteDir()
+                  unstash 'source'
+                  dir("${BASE_DIR}"){
+                    withMageEnv() {
+                      sh(label: 'make release-manager-snapshot', script: 'make release-manager-snapshot')
+                    }
+                  }
+                }
+              }
+              stage('Publish') {
+                steps {
+                  // Copy those files to another location with the sha commit to test them afterward.
+                  googleStorageUpload(bucket: "gs://${JOB_GCS_BUCKET}/${URI_SUFFIX}",
+                    credentialsId: "${JOB_GCS_CREDENTIALS}",
+                    pathPrefix: "${BASE_DIR}/build/distributions/",
+                    pattern: "${BASE_DIR}/build/distributions/**/*",
+                    sharedPublicly: true,
+                    showInline: true)
+                  // Copy the dependencies files if no ARM
+                  whenFalse(isArm()) {
+                    googleStorageUpload(bucket: "gs://${JOB_GCS_BUCKET}/${URI_SUFFIX}",
+                      credentialsId: "${JOB_GCS_CREDENTIALS}",
+                      pathPrefix: "${BASE_DIR}/build/",
+                      pattern: "${BASE_DIR}/build/dependencies.csv",
+                      sharedPublicly: true,
+                      showInline: true)
+                  }
+                }
+              }
+            }
+          }
+        }
+        stage('DRA') {
+          steps {
+            googleStorageDownload(bucketUri: "gs://${JOB_GCS_BUCKET}/${URI_SUFFIX}/*",
+                                  credentialsId: "${JOB_GCS_CREDENTIALS}",
+                                  localDirectory: "${BASE_DIR}/build/distributions",
+                                  pathPrefix: env.PATH_PREFIX)
+            dir("${BASE_DIR}") {
+              dockerLogin(secret: env.DOCKER_SECRET, registry: env.DOCKER_REGISTRY)
+              script {
+                getVaultSecret.readSecretWrapper {
+                  sh(label: 'release-manager.sh', script: '.ci/scripts/release-manager.sh')
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult()
+    }
+    failure {
+      notifyStatus(slackStatus: 'danger', subject: "[${env.REPO}] DRA failed", body: "Build: (<${env.RUN_DISPLAY_URL}|here>)")
+    }
+  }
+}
+
+def notifyStatus(def args = [:]) {
+  releaseNotification(slackChannel: "${env.SLACK_CHANNEL}",
+                      slackColor: args.slackStatus,
+                      slackCredentialsId: 'jenkins-slack-integration-token',
+                      to: "${env.NOTIFY_TO}",
+                      subject: args.subject,
+                      body: args.body)
+}

.ci/packer_cache.sh

@@ -17,11 +17,12 @@ docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian8
 docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-mips
 docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-ppc
 docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-s390x
+docker.elastic.co/infra/release-manager:latest
 golang:${GO_VERSION}
 "
 if [ -x "$(command -v docker)" ]; then
   for image in ${DOCKER_IMAGES}
   do
-  (retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image, we continue"
+  (retry 2 docker pull "${image}") || echo "Error pulling ${image} Docker image, we continue"
   done
 fi

.ci/scripts/package-docker-snapshot.sh

@@ -11,7 +11,8 @@ set -euo pipefail
 NEW_TAG=${1:?Docker tag is not set}
 NEW_IMAGE=${2:?Docker image is not set}
 
-export PLATFORMS='linux/amd64'
+# linux/amd64 is in the default list already
+export PLATFORMS="${PLATFORMS:-+linux/amd64}"
 export TYPE='docker'
 export SNAPSHOT='true'
 export IMAGE="docker.elastic.co/apm/apm-server"

.ci/scripts/release-manager.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+#
+# This script is executed by the release snapshot stage.
+# It requires the below environment variables:
+# - BRANCH_NAME
+# - VAULT_ADDR
+# - VAULT_ROLE_ID
+# - VAULT_SECRET_ID
+#
+set -uexo pipefail
+
+# set required permissions on artifacts and directory
+chmod -R a+r build/distributions/*
+chmod -R a+w build/distributions
+
+# rename docker files to support the unified release format.
+# TODO: this could be supported by the package system itself
+#       or the unified release process the one to do the transformation
+for i in build/distributions/*linux-arm64.docker.tar.gz*
+do
+    mv "$i" "${i/linux-arm64.docker.tar.gz/docker-image-arm64.tar.gz}"
+done
+
+for i in build/distributions/*linux-amd64.docker.tar.gz*
+do
+    mv "$i" "${i/linux-amd64.docker.tar.gz/docker-image.tar.gz}"
+done
+
+# ensure the latest image has been pulled
+IMAGE=docker.elastic.co/infra/release-manager:latest
+docker pull --quiet $IMAGE
+
+# Generate checksum files and upload to GCS
+docker run --rm \
+  --name release-manager \
+  -e VAULT_ADDR \
+  -e VAULT_ROLE_ID \
+  -e VAULT_SECRET_ID \
+  --mount type=bind,readonly=false,src="$PWD",target=/artifacts \
+  "$IMAGE" \
+    cli collect \
+      --project apm-server \
+      --branch "$BRANCH_NAME" \
+      --commit "$(git rev-parse HEAD)" \
+      --workflow "snapshot" \
+      --artifact-set main