diff --git a/src/test/groovy/ApmBasePipelineTest.groovy b/src/test/groovy/ApmBasePipelineTest.groovy index 4cb9542b3..607939650 100644 --- a/src/test/groovy/ApmBasePipelineTest.groovy +++ b/src/test/groovy/ApmBasePipelineTest.groovy @@ -233,6 +233,7 @@ class ApmBasePipelineTest extends DeclarativePipelineTest { helper.registerAllowedMethod('timestamps', [], null) helper.registerAllowedMethod('triggers', [Closure.class], null) helper.registerAllowedMethod('unstable', [Closure.class], { body -> body() }) + helper.registerAllowedMethod('waitUntil', [Map.class, Closure.class], { m, body -> body() }) } void registerScriptedMethods() { diff --git a/src/test/groovy/PushDockerImagesStepTests.groovy b/src/test/groovy/PushDockerImagesStepTests.groovy new file mode 100644 index 000000000..8031d72f9 --- /dev/null +++ b/src/test/groovy/PushDockerImagesStepTests.groovy @@ -0,0 +1,126 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Before +import org.junit.Test +import static org.junit.Assert.assertTrue + +class PushDockerImagesStepTests extends ApmBasePipelineTest { + + @Override + @Before + void setUp() throws Exception { + super.setUp() + script = loadScript('vars/pushDockerImages.groovy') + env.GIT_BASE_COMMIT = 'commit' + } + + @Test + void test_windows() throws Exception { + testWindows() { + script.call() + } + } + + @Test + void test_missing_registry() throws Exception { + testMissingArgument('registry') { + script.call() + } + } + + @Test + void test_missing_secret() throws Exception { + testMissingArgument('secret') { + script.call(registry: 'foo') + } + } + + @Test + void test_missing_version() throws Exception { + testMissingArgument('version') { + script.call(registry: 'foo', secret: 'bar', targetNamespace: 'target') + } + } + + @Test + void test_calculateTags_pr() throws Exception { + helper.registerAllowedMethod('isPR', { return true }) + env.CHANGE_ID = '1' + def ret = script.calculateTags('8.2-SNAPSHOT', '') + printCallStack() + assertTrue(ret.equals(['commit', 'pr-1'])) + } + + @Test + void test_calculateTags_branch() throws Exception { + helper.registerAllowedMethod('isPR', { return false }) + def ret = script.calculateTags('8.2-SNAPSHOT', '8.2.0-SNAPSHOT') + printCallStack() + assertTrue(ret.equals(['commit', '8.2-SNAPSHOT', '8.2.0-SNAPSHOT'])) + + ret = script.calculateTags('8.2-SNAPSHOT', '') + printCallStack() + assertTrue(ret.equals(['commit', '8.2-SNAPSHOT'])) + } + + @Test + void test_doTagAndPush() throws Exception { + helper.registerAllowedMethod('sh', [Map.class], { return 0 }) + script.doTagAndPush(registry: 'my-registry', + sourceTag: '8.2-SNAPSHOT', + targetTag: 'commit', + source: 'beats/my-name-cloud', + target: 'beats-ci/my-name-cloud') + printCallStack() + assertTrue(assertMethodCallContainsPattern('sh', '"my-registry/beats/my-name-cloud:8.2-SNAPSHOT" "my-registry/beats-ci/my-name-cloud:commit"')) + } + + @Test + void test_with_snapshots() throws Exception { + helper.registerAllowedMethod('sh', [Map.class], { return 0 }) + script.call( + secret: "my-secret", + registry: "my-registry", + version: '8.2.0', + snapshot: true, + images: [ + [ source: "beats/filebeat", arch: 'amd64', target: "observability-ci/filebeat"], + [ source: "beats-ci/filebeat-cloud", arch: 'amd64', target: "observability-ci/filebeat-cloud"] + ] + ) + printCallStack() + assertTrue(assertMethodCallOccurrences('sh', 12)) + } + + @Test + void test_without_snapshots() throws Exception { + helper.registerAllowedMethod('sh', [Map.class], { return 0 }) + script.call( + secret: "my-secret", + registry: "my-registry", + version: '8.2.0', + snapshot: false, + images: [ + [ source: "beats/filebeat", arch: 'amd64', target: "observability-ci/filebeat"], + [ source: "beats-ci/filebeat-cloud", arch: 'amd64', target: "observability-ci/filebeat-cloud"] + ] + ) + printCallStack() + assertTrue(assertMethodCallOccurrences('sh', 8)) + } +} diff --git a/vars/README.md b/vars/README.md index 441cc9431..a32a1fb7c 100644 --- a/vars/README.md +++ b/vars/README.md @@ -184,6 +184,7 @@ Override the `build` step to highlight in BO the URL to the downstream job. ``` build(job: 'foo', parameters: [string(name: "my param", value: some_value)]) +build 'foo' ``` See https://jenkins.io/doc/pipeline/steps/pipeline-build-step/#build-build-a-job @@ -194,9 +195,11 @@ Builds the Docker image for Kibana, from a branch or a pull Request. ``` buildKibanaDockerImage(refspec: 'main') buildKibanaDockerImage(refspec: 'PR/12345') +buildKibanaDockerImage(refspec: 'cf25ac3d1f8edff8f20003add707bfdc85d89fff', depth: 10) + ``` -* refspec: A branch (i.e. main), or a pull request identified by the "pr/" prefix and the pull request ID. +* refspec: A branch (i.e. main), a commit SHA, a tag, or a pull request identified by the "pr/" prefix and the pull request ID. * packageJSON: Full name of the package.json file. Defaults to 'package.json' * baseDir: Directory where to clone the Kibana repository. Defaults to "${env.BASE_DIR}/build" * credentialsId: Credentials used access Github repositories. @@ -205,6 +208,9 @@ buildKibanaDockerImage(refspec: 'PR/12345') * dockerRegistrySecret: Name of the Vault secret with the credentials for logining into the registry. Defaults to 'secret/observability-team/ci/docker-registry/prod' * dockerImageSource: Name of the source Docker image when tagging. Defaults to '${dockerRegistry}/kibana/kibana' * dockerImageTarget: Name of the target Docker image to be tagged. Defaults to '${dockerRegistry}/observability-ci/kibana' +* reference: Path to the Git reference repo to improve checkout speed. Default to '/var/lib/jenkins/kibana.git' +* depth: Number of commits pull down in the Git shallow clone. Default to 1 +* shallow: Enable shallow cloning. Default to true. ## buildStatus Fetch the current build status for a given job @@ -667,7 +673,7 @@ update the name of the branch when a new minor release branch is created. ``` // Return the branch name for the main, 8.minor and 8.next-minor branches -def branches = getBranchesFromAliases(aliases: ['main', '8.', '8.']) +def branches = getBranchesFromAliases(aliases: ['main', '8.', '8.']) ``` @@ -1543,6 +1549,7 @@ Upload the given pattern files to the given bucket. * credentialsId: The credentials to access the repo (repo permissions). Optional. Default to `JOB_GCS_CREDENTIALS` * pattern: The file to pattern to search and copy. Mandatory. * sharedPublicly: Whether to shared those objects publicly. Optional. Default false. +* extraFlags: Extra flags to use with gsutil cp. Optional ## gsutil Wrapper to interact with the gsutil command line. It returns the stdout output. @@ -2526,6 +2533,66 @@ with the given headers. __NOTE__: It requires *Nix where to run it from. +## pushDockerImages +Publish docker images in the given docker registry. For such, it +retags the existing docker images and publish them in the given +docker namespace. + +It uses a map of images, this map contains an entry for each docker image +to be pushed, what architecture and the name of the docker image to be pushed. + +The version is required to tag the docker image accordingly and also it uses +the snapshot if needed. + +``` + // Given the filebeat project, and its generated docker + // images for the 8.2.0-SNAPSHOT and 2 different variants + // then publish them to the observability-ci namespace. In addition + // tag them as default, arch=amd64 is the default tag image + pushDockerImages( + registry: "my-registry", + secret: "my-secret", + version: '8.2.0', + snapshot: true, + images: [ + [ source: "beats/filebeat", arch: 'amd64', target: "observability-ci/filebeat"], + [ source: "beats/filebeat-ubi8", arch: 'amd64', target: "observability-ci/filebeat-ubi8"] + ] + ) +``` + +``` + // Given the filebeat project, and its generated docker + // images for the 8.2.0-SNAPSHOT and 2 different variants + // then publish them to observability-ci + // Source images follow the format: + // - "my-registry/beats/filebeat:8.2.0-SNAPSHOT" + // - "my-registry/beats-ci/filebeat-cloud:8.2.0-SNAPSHOT" + // Generated images follow the format: + // - "my-registry/observability-ci/filebeat:8.2.0-SNAPSHOT" + // - "my-registry/observability-ci/filebeat-cloud:8.2.0-SNAPSHOT" + // - "my-registry/observability-ci/filebeat:8.2.0-SNAPSHOT-amd64" + // - "my-registry/observability-ci/filebeat-cloud:8.2.0-SNAPSHOT-amd64" + pushDockerImages( + registry: "my-registry", + secret: "my-secret", + version: '8.2.0', + snapshot: true, + images: [ + [ source: "beats/filebeat", arch: 'amd64', target: "observability-ci/filebeat"], + [ source: "beats-ci/filebeat-cloud", arch: 'amd64', target: "observability-ci/filebeat-ubi8"] + ] + ) +``` + +* secret: the docker secret +* registry: the docker registry +* version: what version +* snapshot: snapshot support +* images: list of the docker image to be retagged to, architecture and docker image to be pushed to. + +__NOTE__: It requires *Nix where to run it from. + ## randomNumber it generates a random number, by default the number is between 1 to 100. @@ -3049,9 +3116,10 @@ not overridden once they are created. ``` * repo: The GitHub repository name. Optional. Default to `REPO` -* bucket: The Google Storage bucket name. Mandatory +* bucket: The Google Storage bucket name. Optional. Default to `JOB_GCS_BUCKET` * credentialsId: The credentials to access the repo (repo permissions). Optional. Default to `JOB_GCS_CREDENTIALS` * pattern: The file to pattern to search and copy. Optional. Default to `build/distributions/**/*` +* folder: The folder to be added to the calculated bucket uri folder. Optional. NOTE: It works with the Multibranch Pipeline only, therefore it requires to use `gitCheckout` to be able to populate the gitBaseCommit. diff --git a/vars/pushDockerImages.groovy b/vars/pushDockerImages.groovy new file mode 100644 index 000000000..a456ec49d --- /dev/null +++ b/vars/pushDockerImages.groovy @@ -0,0 +1,120 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/** + +Publish the give docker images in the given docker registry. For such, it +retags the existing docker images and publish them in the given +docker namespace. + + pushDockerImages( + registry: "my-registry", + secret: "my-secret", + version: '8.2.0', + snapshot: true, + images: [ + [ source: "beats/filebeat", arch: 'amd64', target: "observability-ci/filebeat"], + [ source: "beats/filebeat-ubi8", arch: 'amd64', target: "observability-ci/filebeat-ubi8"], + [ source: "beats-ci/filebeat-cloud", arch: 'amd64', target: "observability-ci/filebeat-cloud"], + [ source: "beats-ci/filebeat-complete", arch: 'amd64', target: "observability-ci/filebeat-complete"] + ] + ) + +*/ +def call(Map args = [:]) { + if(!isUnix()){ + error('pushDockerImages: windows is not supported yet.') + } + def registry = args.containsKey('registry') ? args.registry : error('pushDockerImages: registry parameter is required') + def secret = args.containsKey('secret') ? args.secret : error('pushDockerImages: secret parameter is required') + def version = args.containsKey('version') ? args.version : error('pushDockerImages: version parameter is required') + def snapshot = args.get('snapshot', true) + def images = args.get('images', [:]) + + // Transform version in a snapshot. + def sourceTag = version + def aliasVersion = "" + if (snapshot) { + // remove third number in version + aliasVersion = version.substring(0, version.lastIndexOf(".")) + "-SNAPSHOT" + sourceTag += "-SNAPSHOT" + } + + // What docker tags are gonna be used + def tags = calculateTags(sourceTag, aliasVersion) + + dockerLogin(secret: "${secret}", registry: "${registry}") + images?.each { image -> + tags.each { tag -> + // TODO: + // For backward compatibility let's ensure we tag only for amd64, then E2E can benefit from until + // they support the versioning with the architecture + if ("${image.arch}" == "amd64") { + doTagAndPush(registry: registry, sourceTag: sourceTag, targetTag: "${tag}", source: image.source, target: image.target) + } + doTagAndPush(registry: registry, sourceTag: sourceTag, targetTag: "${tag}-${image.arch}", source: image.source, target: image.target) + } + } +} + +/** +* Tag and push the source docker image. It retries to add resilience. +* +* @param source the namespace and docker image to be used +* @param target the namespace and docker image to be pushed +* @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace +* @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace +* @param registry the docker registry +*/ +def doTagAndPush(Map args = [:]) { + def registry = args.registry + def source = args.source + def sourceTag = args.sourceTag + def target = args.target + def targetTag = args.targetTag + + def sourceName = "${registry}/${source}:${sourceTag}" + def targetName = "${registry}/${target}:${targetTag}" + def iterations = 0 + + waitUntil(initialRecurrencePeriod: 5000) { + iterations++ + def status = sh(label: "Change tag and push ${targetName}", + script: """#!/bin/bash + set -e + echo "source: '${sourceName}' target: '${targetName}'" + if docker image inspect "${sourceName}" &> /dev/null ; then + docker tag "${sourceName}" "${targetName}" + docker push "${targetName}" + else + echo "docker image ${sourceName} does not exist" + fi""", + returnStatus: true) + // exit if above command run successfully or it reached the max of iterations. + return (status == 0 || iterations >= 3) + } +} + +def calculateTags(sourceTag, aliasVersion) { + def tags = [ env.GIT_BASE_COMMIT, + isPR() ? "pr-${env.CHANGE_ID}" : sourceTag ] + + if (!isPR() && aliasVersion.trim()) { + tags << aliasVersion + } + return tags +} diff --git a/vars/pushDockerImages.txt b/vars/pushDockerImages.txt new file mode 100644 index 000000000..b348684ff --- /dev/null +++ b/vars/pushDockerImages.txt @@ -0,0 +1,58 @@ +Publish docker images in the given docker registry. For such, it +retags the existing docker images and publish them in the given +docker namespace. + +It uses a map of images, this map contains an entry for each docker image +to be pushed, what architecture and the name of the docker image to be pushed. + +The version is required to tag the docker image accordingly and also it uses +the snapshot if needed. + +``` + // Given the filebeat project, and its generated docker + // images for the 8.2.0-SNAPSHOT and 2 different variants + // then publish them to the observability-ci namespace. In addition + // tag them as default, arch=amd64 is the default tag image + pushDockerImages( + registry: "my-registry", + secret: "my-secret", + version: '8.2.0', + snapshot: true, + images: [ + [ source: "beats/filebeat", arch: 'amd64', target: "observability-ci/filebeat"], + [ source: "beats/filebeat-ubi8", arch: 'amd64', target: "observability-ci/filebeat-ubi8"] + ] + ) +``` + +``` + // Given the filebeat project, and its generated docker + // images for the 8.2.0-SNAPSHOT and 2 different variants + // then publish them to observability-ci + // Source images follow the format: + // - "my-registry/beats/filebeat:8.2.0-SNAPSHOT" + // - "my-registry/beats-ci/filebeat-cloud:8.2.0-SNAPSHOT" + // Generated images follow the format: + // - "my-registry/observability-ci/filebeat:8.2.0-SNAPSHOT" + // - "my-registry/observability-ci/filebeat-cloud:8.2.0-SNAPSHOT" + // - "my-registry/observability-ci/filebeat:8.2.0-SNAPSHOT-amd64" + // - "my-registry/observability-ci/filebeat-cloud:8.2.0-SNAPSHOT-amd64" + pushDockerImages( + registry: "my-registry", + secret: "my-secret", + version: '8.2.0', + snapshot: true, + images: [ + [ source: "beats/filebeat", arch: 'amd64', target: "observability-ci/filebeat"], + [ source: "beats-ci/filebeat-cloud", arch: 'amd64', target: "observability-ci/filebeat-ubi8"] + ] + ) +``` + +* secret: the docker secret +* registry: the docker registry +* version: what version +* snapshot: snapshot support +* images: list of the docker image to be retagged to, architecture and docker image to be pushed to. + +__NOTE__: It requires *Nix where to run it from.