Add googleStorageUploadExt step

.ci/Jenkinsfile

@@ -28,6 +28,8 @@ pipeline {
     LANG = "C.UTF-8"
     LC_ALL = "C.UTF-8"
     SLACK_CHANNEL = '#observablt-bots'
+    JOB_GCS_BUCKET = 'apm-ci-temp'
+    JOB_GCS_CREDENTIALS = 'apm-ci-gcs-plugin-file-credentials'
     GITHUB_CHECK = 'true'
   }
   options {
@@ -114,6 +116,18 @@ pipeline {
         }
       }
     }
+    stage('Google Storage') {
+      when {
+        branch 'master'  // TODO: to be removed as soon as the step is available in the master branch
+      }
+      steps {
+        withGithubNotify(context: 'Google Storage', description: 'Google Storage step') {
+          dir("${BASE_DIR}"){
+            googleStorageUploadExt(bucket: "gs://${env.JOB_GCS_BUCKET}/${env.REPO}/commits/${env.GIT_BASE_COMMIT}", pattern: 'target/**/*.xml', sharedPublicly: false)
+          }
+        }
+      }
+    }
     stage('Check Pipelines') {
       steps {
         withGithubNotify(context: 'Check Pipelines') {

docs/STORAGE.md

@@ -0,0 +1,21 @@
+# Storage
+
+We are using the Google Storage Jenkins plugins but it is disabled for the time being because it has some issues with some parallelisation.
+
+In order to test this `locally`, we have configured some vault credentials. See below if you need to change them:
+
+## How to
+
+### Modify the existing credentials for the google service account
+
+If for any reason you'd like to update the existing credentials then you need to follow the below steps:
+
+1. Go to [Service Accounts](https://console.cloud.google.com/iam-admin/serviceaccounts?project=elastic-observability)
+1. Select the `test-google-storage-plugin-download` one.
+1. Create a new JSON key type.
+1. Transform to base64 and update the existing vault entry
+
+```bash
+$ base64 -i elastic-observability-*********.json -o base64.json
+$ vault write secret/observability-team/ci/service-account/jenkins-google-storage-elastic-observability google_cloud_bucket_secret=@base64.json ticket=https://github.com/elastic/apm-pipeline-library/pull/867
+```

local/configs/jenkins.yaml

@@ -73,6 +73,12 @@ credentials:
               id: gcs-bucket
               scope: GLOBAL
               secret: ${gcs_bucket}
+          - file: ## IMPORTANT This credential is only for testing the googleStorageUploadExt step since it uses some specific service account
+              description: "Secret File Description for apm-ci-gcs-plugin"
+              fileName: "elastic-observability.json"
+              id: "apm-ci-gcs-plugin-file-credentials"
+              scope: GLOBAL
+              secretBytes: "${google_cloud_bucket_secret}" # secretBytes requires base64 encoded content
 
 unclassified:
   location:
@@ -106,6 +112,7 @@ unclassified:
   openTelemetry:
     endpoint: "otel-collector-contrib:4317"
     useTls: false
+
 jobs:
   - file: "/var/pipeline-library/src/test/resources/folders/it.dsl"
   - file: "/var/pipeline-library/src/test/resources/folders/beats.dsl"
@@ -133,6 +140,7 @@ jobs:
   - file: "/var/pipeline-library/src/test/resources/jobs/githubCreatePullRequest.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/githubEnv.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/githubEnvSCM.dsl"
+  - file: "/var/pipeline-library/src/test/resources/jobs/googleStorageUploadExt.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/installTools.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/isTimerTrigger.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/isUserTrigger.dsl"

local/docker-compose.yml

@@ -8,7 +8,7 @@ services:
       CASC_JENKINS_CONFIG: /var/jenkins_home/casc_configs
       CASC_VAULT_ENGINE_VERSION: "1"
       CASC_VAULT_PATHS: |
-        secret/jcasc/localhost/base,secret/jcasc/localhost/apm-ci
+        secret/jcasc/localhost/base,secret/jcasc/localhost/apm-ci,secret/observability-team/ci/service-account/jenkins-google-storage-elastic-observability
       CASC_VAULT_TOKEN: ${VAULT_TOKEN}
       CASC_VAULT_URL: ${VAULT_ADDR:-https://secrets.elastic.co:8200}
       JAVA_OPTS: >-

src/test/groovy/ApmBasePipelineTest.groovy

@@ -274,6 +274,7 @@ class ApmBasePipelineTest extends DeclarativePipelineTest {
       updateBuildStatus('FAILURE')
       throw new Exception(s)
     })
+    helper.registerAllowedMethod('file', [Map.class], { [ variable: 'foo', secret: 'bar' ] })
     helper.registerAllowedMethod('fileExists', [String.class], { true })
     helper.registerAllowedMethod('fileExists', [Map.class], { true })
     helper.registerAllowedMethod('getContext', [org.jenkinsci.plugins.workflow.graph.FlowNode.class], null)
@@ -451,6 +452,10 @@ class ApmBasePipelineTest extends DeclarativePipelineTest {
       }
       return ret
     })
+    helper.registerAllowedMethod('gsutil', [Map.class], { m ->
+      def script = loadScript('vars/gsutil.groovy')
+      return script.call(m)
+    })
     helper.registerAllowedMethod('httpRequest', [Map.class], { true })
     helper.registerAllowedMethod('installTools', [List.class], { l ->
       def script = loadScript('vars/installTools.groovy')
@@ -492,6 +497,7 @@ class ApmBasePipelineTest extends DeclarativePipelineTest {
       def script = loadScript('vars/is32x86.groovy')
       return script.call()
     })
+    helper.registerAllowedMethod('is64', { return true })
     helper.registerAllowedMethod('is64x86', {
       def script = loadScript('vars/is64x86.groovy')
       return script.call()

src/test/groovy/GoogleStorageUploadExtStepTests.groovy

@@ -0,0 +1,99 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Before
+import org.junit.Test
+import static org.junit.Assert.assertFalse
+import static org.junit.Assert.assertTrue
+
+class GoogleStorageUploadExtStepTests extends ApmBasePipelineTest {
+  def script
+
+  @Override
+  @Before
+  void setUp() throws Exception {
+    super.setUp()
+    env.JOB_GCS_CREDENTIALS = 'secret'
+    script = loadScript('vars/googleStorageUploadExt.groovy')
+  }
+
+  @Test
+  void test_windows() throws Exception {
+    helper.registerAllowedMethod('isUnix', [], { false })
+    try {
+      script.call(bucket: 'gs://foo', pattern: 'file.txt')
+    } catch(e){
+      //NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'gsutil: windows is not supported yet.'))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_without_bucket() throws Exception {
+    try {
+      script.call()
+    } catch(err) {
+      // NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'googleStorageUploadExt: bucket parameter is required'))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_without_pattern() throws Exception {
+    try {
+      script.call(bucket: 'gs://foo')
+    } catch(err) {
+      // NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'googleStorageUploadExt: pattern parameter is required'))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_with_gsutil_error() throws Exception {
+    helper.registerAllowedMethod('gsutil', [Map.class], { throw new Exception('unknown command "foo" for "gsutil"') })
+    try {
+      script.call(bucket: 'gs://foo', pattern: 'file.txt')
+    } catch(err) {
+      // NOOP
+    }
+    printCallStack()
+  }
+
+  @Test
+  void test() throws Exception {
+    helper.registerAllowedMethod('gsutil', [Map.class], { return 'Operation completed over 1 objects.' })
+    def ret = script.call(bucket: 'gs://foo', pattern: 'file.txt')
+    printCallStack()
+    assertFalse(assertMethodCallContainsPattern('gsutil', '-a public-read'))
+    assertTrue(assertMethodCallContainsPattern('gsutil', 'file.txt gs://foo'))
+    assertFalse(ret.isEmpty())
+  }
+
+  @Test
+  void test_with_shared_publically() throws Exception {
+    helper.registerAllowedMethod('gsutil', [Map.class], { return 'Operation completed over 1 objects.' })
+    script.call(bucket: 'gs://foo', pattern: 'file.txt', sharedPublicly: true)
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('gsutil', '-a public-read file.txt gs://foo'))
+  }
+}

src/test/groovy/GsutilStepTests.groovy

@@ -0,0 +1,145 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Before
+import org.junit.Test
+import static org.junit.Assert.assertFalse
+import static org.junit.Assert.assertNull
+import static org.junit.Assert.assertTrue
+
+class GsutilStepTests extends ApmBasePipelineTest {
+  def script
+
+  @Override
+  @Before
+  void setUp() throws Exception {
+    super.setUp()
+    helper.registerAllowedMethod('isInstalled', [Map.class], { return true })
+    script = loadScript('vars/gsutil.groovy')
+  }
+
+  @Test
+  void test_windows() throws Exception {
+    helper.registerAllowedMethod('isUnix', [], { false })
+    try {
+      script.call()
+    } catch(e){
+      //NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'gsutil: windows is not supported yet.'))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_without_command() throws Exception {
+    try {
+      script.call()
+    } catch(e) {
+      // NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'gsutil: command argument is required'))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_without_credentials() throws Exception {
+    try {
+      script.call(command: 'cp')
+    } catch(e) {
+      // NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'gsutil: credentialsId argument is required.'))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_command() throws Exception {
+    script.call(command: 'cp', credentialsId: 'foo')
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('withCredentials', ''))
+    assertTrue(assertMethodCallContainsPattern('sh', "gsutil cp"))
+    assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL'))
+    assertFalse(assertMethodCallContainsPattern('sh', "wget -q -O"))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void test_with_failed() throws Exception {
+    helper.registerAllowedMethod('sh', [Map.class], { m ->
+      if (m.label.startsWith('gsutil')) { throw new Exception('unknown command "foo" for "gsutil"') }})
+    def result
+    try {
+      result = script.call(command: 'foo', credentialsId: 'foo')
+    } catch(err) {
+      println err
+      // NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('sh', 'returnStdout=true'))
+    assertNull(result)
+  }
+
+  @Test
+  void test_without_gh_installed_by_default_with_wget() throws Exception {
+    helper.registerAllowedMethod('isInstalled', [Map.class], { m -> return m.tool.equals('wget') })
+    script.call(command: 'cp', credentialsId: 'foo')
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'wget -q -O'))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void test_without_gh_installed_by_default_no_wget() throws Exception {
+    helper.registerAllowedMethod('isInstalled', [Map.class], { return false })
+    script.call(command: 'cp', credentialsId: 'foo')
+    printCallStack()
+    assertFalse(assertMethodCallContainsPattern('sh', 'wget -q -O'))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void test_cache() throws Exception {
+    helper.registerAllowedMethod('isInstalled', [Map.class], { m -> return m.tool.equals('wget') })
+    try {
+      script.call(command: 'cp', credentialsId: 'foo')
+      script.call(command: 'cp', credentialsId: 'foo')
+    } catch(e) {
+      // NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'wget -q -O'))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void test_cache_without_gsutil_installed_by_default_with_wget() throws Exception {
+    helper.registerAllowedMethod('isInstalled', [Map.class], { m -> return m.tool.equals('wget') })
+    script.call(command: 'cp', credentialsId: 'foo')
+    script.call(command: 'cp', credentialsId: 'foo')
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'wget -q -O'))
+    assertTrue(assertMethodCallContainsPattern('log', 'gsutil: get the gsutilLocation from cache.'))
+    assertTrue(assertMethodCallContainsPattern('log', 'gsutil: set the gsutilLocation.'))
+    assertJobStatusSuccess()
+  }
+}

src/test/resources/jobs/googleStorageUploadExt.dsl

@@ -0,0 +1,25 @@
+NAME = 'it/googleStorageUploadExt'
+DSL = '''pipeline {
+  agent { label "master" }
+  environment {
+    JOB_GCS_BUCKET = 'apm-ci-temp'
+    JOB_GCS_CREDENTIALS = 'apm-ci-gcs-plugin-file-credentials'
+    PIPELINE_LOG_LEVEL = 'DEBUG'
+  }
+  stages {
+    stage('google-storage') {
+      steps {
+        touch file: 'file.txt', timestamp: 0
+        googleStorageUploadExt(bucket: "gs://${env.JOB_GCS_BUCKET}/test-${env.BUILD_ID}/", pattern: 'file.txt', sharedPublicly: true)
+      }
+    }
+  }
+}'''
+
+pipelineJob(NAME) {
+  definition {
+    cps {
+      script(DSL.stripIndent())
+    }
+  }
+}