diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index a43c3d381..e9f19d145 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -28,6 +28,8 @@ pipeline { LANG = "C.UTF-8" LC_ALL = "C.UTF-8" SLACK_CHANNEL = '#observablt-bots' + JOB_GCS_BUCKET = 'apm-ci-temp' + JOB_GCS_CREDENTIALS = 'apm-ci-gcs-plugin-file-credentials' GITHUB_CHECK = 'true' } options { @@ -115,6 +117,18 @@ pipeline { } } } + stage('Google Storage') { + when { + branch 'master' // TODO: to be removed as soon as the step is available in the master branch + } + steps { + withGithubNotify(context: 'Google Storage', description: 'Google Storage step') { + dir("${BASE_DIR}"){ + googleStorageUploadExt(bucket: "gs://${env.JOB_GCS_BUCKET}/${env.REPO}/commits/${env.GIT_BASE_COMMIT}", pattern: 'target/**/*.xml', sharedPublicly: false) + } + } + } + } stage('Check Pipelines') { steps { withGithubNotify(context: 'Check Pipelines') { diff --git a/docs/STORAGE.md b/docs/STORAGE.md new file mode 100644 index 000000000..b2adc45a9 --- /dev/null +++ b/docs/STORAGE.md @@ -0,0 +1,21 @@ +# Storage + +We are using the Google Storage Jenkins plugins but it is disabled for the time being because it has some issues with some parallelisation. + +In order to test this `locally`, we have configured some vault credentials. See below if you need to change them: + +## How to + +### Modify the existing credentials for the google service account + +If for any reason you'd like to update the existing credentials then you need to follow the below steps: + +1. Go to [Service Accounts](https://console.cloud.google.com/iam-admin/serviceaccounts?project=elastic-observability) +1. Select the `test-google-storage-plugin-download` one. +1. Create a new JSON key type. +1. Transform to base64 and update the existing vault entry + +```bash +$ base64 -i elastic-observability-*********.json -o base64.json +$ vault write secret/observability-team/ci/service-account/jenkins-google-storage-elastic-observability google_cloud_bucket_secret=@base64.json ticket=https://github.com/elastic/apm-pipeline-library/pull/867 +``` diff --git a/local/configs/jenkins.yaml b/local/configs/jenkins.yaml index 4e57b766d..1d26ac24c 100644 --- a/local/configs/jenkins.yaml +++ b/local/configs/jenkins.yaml @@ -73,6 +73,12 @@ credentials: id: gcs-bucket scope: GLOBAL secret: ${gcs_bucket} + - file: ## IMPORTANT This credential is only for testing the googleStorageUploadExt step since it uses some specific service account + description: "Secret File Description for apm-ci-gcs-plugin" + fileName: "elastic-observability.json" + id: "apm-ci-gcs-plugin-file-credentials" + scope: GLOBAL + secretBytes: "${google_cloud_bucket_secret}" # secretBytes requires base64 encoded content unclassified: location: @@ -106,6 +112,7 @@ unclassified: openTelemetry: endpoint: "otel-collector-contrib:4317" useTls: false + jobs: - file: "/var/pipeline-library/src/test/resources/folders/it.dsl" - file: "/var/pipeline-library/src/test/resources/folders/beats.dsl" @@ -133,6 +140,7 @@ jobs: - file: "/var/pipeline-library/src/test/resources/jobs/githubCreatePullRequest.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/githubEnv.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/githubEnvSCM.dsl" + - file: "/var/pipeline-library/src/test/resources/jobs/googleStorageUploadExt.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/installTools.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/isTimerTrigger.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/isUserTrigger.dsl" diff --git a/local/docker-compose.yml b/local/docker-compose.yml index 90084cb21..17800db3f 100644 --- a/local/docker-compose.yml +++ b/local/docker-compose.yml @@ -8,7 +8,7 @@ services: CASC_JENKINS_CONFIG: /var/jenkins_home/casc_configs CASC_VAULT_ENGINE_VERSION: "1" CASC_VAULT_PATHS: | - secret/jcasc/localhost/base,secret/jcasc/localhost/apm-ci + secret/jcasc/localhost/base,secret/jcasc/localhost/apm-ci,secret/observability-team/ci/service-account/jenkins-google-storage-elastic-observability CASC_VAULT_TOKEN: ${VAULT_TOKEN} CASC_VAULT_URL: ${VAULT_ADDR:-https://secrets.elastic.co:8200} JAVA_OPTS: >- diff --git a/src/test/groovy/ApmBasePipelineTest.groovy b/src/test/groovy/ApmBasePipelineTest.groovy index 053b851ca..fe1419d69 100644 --- a/src/test/groovy/ApmBasePipelineTest.groovy +++ b/src/test/groovy/ApmBasePipelineTest.groovy @@ -274,6 +274,7 @@ class ApmBasePipelineTest extends DeclarativePipelineTest { updateBuildStatus('FAILURE') throw new Exception(s) }) + helper.registerAllowedMethod('file', [Map.class], { [ variable: 'foo', secret: 'bar' ] }) helper.registerAllowedMethod('fileExists', [String.class], { true }) helper.registerAllowedMethod('fileExists', [Map.class], { true }) helper.registerAllowedMethod('getContext', [org.jenkinsci.plugins.workflow.graph.FlowNode.class], null) @@ -451,6 +452,10 @@ class ApmBasePipelineTest extends DeclarativePipelineTest { } return ret }) + helper.registerAllowedMethod('gsutil', [Map.class], { m -> + def script = loadScript('vars/gsutil.groovy') + return script.call(m) + }) helper.registerAllowedMethod('httpRequest', [Map.class], { true }) helper.registerAllowedMethod('installTools', [List.class], { l -> def script = loadScript('vars/installTools.groovy') @@ -492,6 +497,7 @@ class ApmBasePipelineTest extends DeclarativePipelineTest { def script = loadScript('vars/is32x86.groovy') return script.call() }) + helper.registerAllowedMethod('is64', { return true }) helper.registerAllowedMethod('is64x86', { def script = loadScript('vars/is64x86.groovy') return script.call() diff --git a/src/test/groovy/GoogleStorageUploadExtStepTests.groovy b/src/test/groovy/GoogleStorageUploadExtStepTests.groovy new file mode 100644 index 000000000..6dc2a077b --- /dev/null +++ b/src/test/groovy/GoogleStorageUploadExtStepTests.groovy @@ -0,0 +1,99 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Before +import org.junit.Test +import static org.junit.Assert.assertFalse +import static org.junit.Assert.assertTrue + +class GoogleStorageUploadExtStepTests extends ApmBasePipelineTest { + def script + + @Override + @Before + void setUp() throws Exception { + super.setUp() + env.JOB_GCS_CREDENTIALS = 'secret' + script = loadScript('vars/googleStorageUploadExt.groovy') + } + + @Test + void test_windows() throws Exception { + helper.registerAllowedMethod('isUnix', [], { false }) + try { + script.call(bucket: 'gs://foo', pattern: 'file.txt') + } catch(e){ + //NOOP + } + printCallStack() + assertTrue(assertMethodCallContainsPattern('error', 'gsutil: windows is not supported yet.')) + assertJobStatusFailure() + } + + @Test + void test_without_bucket() throws Exception { + try { + script.call() + } catch(err) { + // NOOP + } + printCallStack() + assertTrue(assertMethodCallContainsPattern('error', 'googleStorageUploadExt: bucket parameter is required')) + assertJobStatusFailure() + } + + @Test + void test_without_pattern() throws Exception { + try { + script.call(bucket: 'gs://foo') + } catch(err) { + // NOOP + } + printCallStack() + assertTrue(assertMethodCallContainsPattern('error', 'googleStorageUploadExt: pattern parameter is required')) + assertJobStatusFailure() + } + + @Test + void test_with_gsutil_error() throws Exception { + helper.registerAllowedMethod('gsutil', [Map.class], { throw new Exception('unknown command "foo" for "gsutil"') }) + try { + script.call(bucket: 'gs://foo', pattern: 'file.txt') + } catch(err) { + // NOOP + } + printCallStack() + } + + @Test + void test() throws Exception { + helper.registerAllowedMethod('gsutil', [Map.class], { return 'Operation completed over 1 objects.' }) + def ret = script.call(bucket: 'gs://foo', pattern: 'file.txt') + printCallStack() + assertFalse(assertMethodCallContainsPattern('gsutil', '-a public-read')) + assertTrue(assertMethodCallContainsPattern('gsutil', 'file.txt gs://foo')) + assertFalse(ret.isEmpty()) + } + + @Test + void test_with_shared_publically() throws Exception { + helper.registerAllowedMethod('gsutil', [Map.class], { return 'Operation completed over 1 objects.' }) + script.call(bucket: 'gs://foo', pattern: 'file.txt', sharedPublicly: true) + printCallStack() + assertTrue(assertMethodCallContainsPattern('gsutil', '-a public-read file.txt gs://foo')) + } +} diff --git a/src/test/groovy/GsutilStepTests.groovy b/src/test/groovy/GsutilStepTests.groovy new file mode 100644 index 000000000..c495dd605 --- /dev/null +++ b/src/test/groovy/GsutilStepTests.groovy @@ -0,0 +1,145 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Before +import org.junit.Test +import static org.junit.Assert.assertFalse +import static org.junit.Assert.assertNull +import static org.junit.Assert.assertTrue + +class GsutilStepTests extends ApmBasePipelineTest { + def script + + @Override + @Before + void setUp() throws Exception { + super.setUp() + helper.registerAllowedMethod('isInstalled', [Map.class], { return true }) + script = loadScript('vars/gsutil.groovy') + } + + @Test + void test_windows() throws Exception { + helper.registerAllowedMethod('isUnix', [], { false }) + try { + script.call() + } catch(e){ + //NOOP + } + printCallStack() + assertTrue(assertMethodCallContainsPattern('error', 'gsutil: windows is not supported yet.')) + assertJobStatusFailure() + } + + @Test + void test_without_command() throws Exception { + try { + script.call() + } catch(e) { + // NOOP + } + printCallStack() + assertTrue(assertMethodCallContainsPattern('error', 'gsutil: command argument is required')) + assertJobStatusFailure() + } + + @Test + void test_without_credentials() throws Exception { + try { + script.call(command: 'cp') + } catch(e) { + // NOOP + } + printCallStack() + assertTrue(assertMethodCallContainsPattern('error', 'gsutil: credentialsId argument is required.')) + assertJobStatusFailure() + } + + @Test + void test_command() throws Exception { + script.call(command: 'cp', credentialsId: 'foo') + printCallStack() + assertTrue(assertMethodCallContainsPattern('withCredentials', '')) + assertTrue(assertMethodCallContainsPattern('sh', "gsutil cp")) + assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL')) + assertFalse(assertMethodCallContainsPattern('sh', "wget -q -O")) + assertJobStatusSuccess() + } + + @Test + void test_with_failed() throws Exception { + helper.registerAllowedMethod('sh', [Map.class], { m -> + if (m.label.startsWith('gsutil')) { throw new Exception('unknown command "foo" for "gsutil"') }}) + def result + try { + result = script.call(command: 'foo', credentialsId: 'foo') + } catch(err) { + println err + // NOOP + } + printCallStack() + assertTrue(assertMethodCallContainsPattern('sh', 'returnStdout=true')) + assertNull(result) + } + + @Test + void test_without_gh_installed_by_default_with_wget() throws Exception { + helper.registerAllowedMethod('isInstalled', [Map.class], { m -> return m.tool.equals('wget') }) + script.call(command: 'cp', credentialsId: 'foo') + printCallStack() + assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL')) + assertTrue(assertMethodCallContainsPattern('sh', 'wget -q -O')) + assertJobStatusSuccess() + } + + @Test + void test_without_gh_installed_by_default_no_wget() throws Exception { + helper.registerAllowedMethod('isInstalled', [Map.class], { return false }) + script.call(command: 'cp', credentialsId: 'foo') + printCallStack() + assertFalse(assertMethodCallContainsPattern('sh', 'wget -q -O')) + assertJobStatusSuccess() + } + + @Test + void test_cache() throws Exception { + helper.registerAllowedMethod('isInstalled', [Map.class], { m -> return m.tool.equals('wget') }) + try { + script.call(command: 'cp', credentialsId: 'foo') + script.call(command: 'cp', credentialsId: 'foo') + } catch(e) { + // NOOP + } + printCallStack() + assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL')) + assertTrue(assertMethodCallContainsPattern('sh', 'wget -q -O')) + assertJobStatusSuccess() + } + + @Test + void test_cache_without_gsutil_installed_by_default_with_wget() throws Exception { + helper.registerAllowedMethod('isInstalled', [Map.class], { m -> return m.tool.equals('wget') }) + script.call(command: 'cp', credentialsId: 'foo') + script.call(command: 'cp', credentialsId: 'foo') + printCallStack() + assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL')) + assertTrue(assertMethodCallContainsPattern('sh', 'wget -q -O')) + assertTrue(assertMethodCallContainsPattern('log', 'gsutil: get the gsutilLocation from cache.')) + assertTrue(assertMethodCallContainsPattern('log', 'gsutil: set the gsutilLocation.')) + assertJobStatusSuccess() + } +} diff --git a/src/test/resources/jobs/googleStorageUploadExt.dsl b/src/test/resources/jobs/googleStorageUploadExt.dsl new file mode 100644 index 000000000..f73e8db88 --- /dev/null +++ b/src/test/resources/jobs/googleStorageUploadExt.dsl @@ -0,0 +1,25 @@ +NAME = 'it/googleStorageUploadExt' +DSL = '''pipeline { + agent { label "master" } + environment { + JOB_GCS_BUCKET = 'apm-ci-temp' + JOB_GCS_CREDENTIALS = 'apm-ci-gcs-plugin-file-credentials' + PIPELINE_LOG_LEVEL = 'DEBUG' + } + stages { + stage('google-storage') { + steps { + touch file: 'file.txt', timestamp: 0 + googleStorageUploadExt(bucket: "gs://${env.JOB_GCS_BUCKET}/test-${env.BUILD_ID}/", pattern: 'file.txt', sharedPublicly: true) + } + } + } +}''' + +pipelineJob(NAME) { + definition { + cps { + script(DSL.stripIndent()) + } + } +} diff --git a/vars/googleStorageUploadExt.groovy b/vars/googleStorageUploadExt.groovy new file mode 100644 index 000000000..782b416fc --- /dev/null +++ b/vars/googleStorageUploadExt.groovy @@ -0,0 +1,31 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +def call(Map args = [:]) { + def credentialsId = args.get('credentialsId', env.JOB_GCS_CREDENTIALS) + def bucket = args.containsKey('bucket') ? args.bucket : error('googleStorageUploadExt: bucket parameter is required') + def pattern = args.containsKey('pattern') ? args.pattern : error('googleStorageUploadExt: pattern parameter is required') + def sharedPublicly = args.get('sharedPublicly', false) + + def flags = '' + + if (sharedPublicly) { + flags = '-a public-read' + } + + return gsutil(command: "-m -q cp ${flags} ${pattern} ${bucket}", credentialsId: credentialsId) +} diff --git a/vars/googleStorageUploadExt.txt b/vars/googleStorageUploadExt.txt new file mode 100644 index 000000000..e568511cb --- /dev/null +++ b/vars/googleStorageUploadExt.txt @@ -0,0 +1,12 @@ +Upload the given pattern files to the given bucket. + +``` + // Copy file.txt into the bucket + googleStorageUploadExt(pattern: 'file.txt', bucket: 'gs://bucket/folder/', credentialsId: 'foo', sharedPublicly: false) + +``` + +* bucket: The Google Storage bucket format gs://bucket/folder/subfolder/. Mandatory +* credentialsId: The credentials to access the repo (repo permissions). Optional. Default to `JOB_GCS_CREDENTIALS` +* pattern: The file to pattern to search and copy. Mandatory. +* sharedPublicly: Whether to shared those objects publically. Optional. Default false. diff --git a/vars/gsutil.groovy b/vars/gsutil.groovy new file mode 100644 index 000000000..a2d0ffea0 --- /dev/null +++ b/vars/gsutil.groovy @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.transform.Field + +@Field def gsUtilLocation = '' + +def call(Map args = [:]) { + if(!isUnix()) { + error 'gsutil: windows is not supported yet.' + } + def command = args.containsKey('command') ? args.command : error('gsutil: command argument is required.') + def credentialsId = args.containsKey('credentialsId') ? args.credentialsId : error('gsutil: credentialsId argument is required.') + + if (gsUtilLocation?.trim()) { + log(level: 'DEBUG', text: 'gsutil: get the gsutilLocation from cache.') + } else { + log(level: 'DEBUG', text: 'gsutil: set the gsutilLocation.') + gsUtilLocation = pwd(tmp: true) + } + + withEnv(["PATH+GSUTIL=${gsUtilLocation}", "PATH+GSUTIL_BIN=${gsUtilLocation}/bin"]) { + if(!isInstalled(tool: 'gsutil', flag: '--version')) { + downloadInstaller(gsUtilLocation) + } + + withCredentials([file(credentialsId: credentialsId, variable: 'FILE_CREDENTIAL')]) { + sh(label: 'authenticate', script: 'gcloud auth activate-service-account --key-file ${FILE_CREDENTIAL}') + } + return sh(label: "gsutil ${command}", script: "gsutil ${command}", returnStdout: true) + } +} + +def downloadInstaller(where) { + def url = 'https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-319.0.0' + url = "${url}-linux-${is64() ? 'x86_64' : 'x86'}.tar.gz" + def tarball = 'gsutil.tar.gz' + if(isInstalled(tool: 'wget', flag: '--version')) { + dir(where) { + retryWithSleep(retries: 3, seconds: 5, backoff: true) { + sh(label: 'download gsutil', script: "wget -q -O ${tarball} ${url}") + sh(label: 'untar gsutil', script: "tar -xpf ${tarball} --strip-components=1") + } + } + } else { + log(level: 'WARN', text: 'gsutil: wget is not available. gsutil will not be installed then.') + } +} diff --git a/vars/gsutil.txt b/vars/gsutil.txt new file mode 100644 index 000000000..05621a8e0 --- /dev/null +++ b/vars/gsutil.txt @@ -0,0 +1,12 @@ +Wrapper to interact with the gsutil command line. It returns the stdout output. + +``` + // Copy file.txt into the bucket + gsutil(command: 'cp file.txt gs://bucket/folder/', credentialsId: 'foo' ]) + +``` + +* command: The gsutil command to be executed. Mandatory +* credentialsId: The credentials to access the repo (repo permissions). Mandatory. + +_NOTE_: Windows is not supported yet.