Support windows for gsutil (bp #985)

.ci/Jenkinsfile

@@ -118,13 +118,25 @@ pipeline {
       }
     }
     stage('Google Storage') {
+      steps {
+        withGithubNotify(context: 'Google Storage', description: 'Google Storage step') {
+          dir("${BASE_DIR}"){
+            googleStorageUploadExt(bucket: "gs://${env.JOB_GCS_BUCKET}/${env.REPO}/commits/${env.GIT_BASE_COMMIT}", pattern: 'src/**/*.gdsl', sharedPublicly: false)
+          }
+        }
+      }
+    }
+    stage('Google Storage Windows') {
+      agent { label 'windows-2019-immutable' }
       when {
-        branch 'master'  // TODO: to be removed as soon as the step is available in the master branch
+        branch 'master'
       }
       steps {
-        withGithubNotify(context: 'Google Storage', description: 'Google Storage step') {
+        withGithubNotify(context: 'Google Storage Windows', description: 'Google Storage step') {
+          deleteDir()
+          unstash 'source'
           dir("${BASE_DIR}"){
-            googleStorageUploadExt(bucket: "gs://${env.JOB_GCS_BUCKET}/${env.REPO}/commits/${env.GIT_BASE_COMMIT}", pattern: 'target/**/*.xml', sharedPublicly: false)
+            googleStorageUploadExt(bucket: "gs://${env.JOB_GCS_BUCKET}/${env.REPO}/commits/${env.GIT_BASE_COMMIT}", pattern: 'resources/**/*.json', sharedPublicly: false)
           }
         }
       }
@@ -136,9 +148,15 @@ pipeline {
             validateDeclarativePipeline '.ci/Jenkinsfile'
             validateDeclarativePipeline '.ci/buildBeatsDockerImages.groovy'
             validateDeclarativePipeline '.ci/buildDockerImages.groovy'
+            validateDeclarativePipeline '.ci/chatops.groovy'
+            validateDeclarativePipeline '.ci/buildDockerImages.groovy'
             validateDeclarativePipeline '.ci/dockerImagesOpbeans.groovy'
+            validateDeclarativePipeline '.ci/jobdsl-generator.groovy'
+            validateDeclarativePipeline '.ci/licenseScan.groovy'
+            validateDeclarativePipeline '.ci/licenseScanGeneral.groovy'
             validateDeclarativePipeline '.ci/schedule-daily.groovy'
             validateDeclarativePipeline '.ci/schedule-weekly.groovy'
+            validateDeclarativePipeline '.ci/validateWorkersBeatsCi.groovy'
             validateDeclarativePipeline 'resources/JenkinsfileTemplate.groovy'
           }
         }

src/test/groovy/ApmBasePipelineTest.groovy

@@ -334,6 +334,7 @@ class ApmBasePipelineTest extends DeclarativePipelineTest {
     helper.registerAllowedMethod('string', [Map.class], { m -> return m })
     helper.registerAllowedMethod('timeout', [Integer.class, Closure.class], null)
     helper.registerAllowedMethod('unstash', [String.class], null)
+    helper.registerAllowedMethod('unzip', [Map.class], null)
     helper.registerAllowedMethod('upstreamDevelopers', { "OK" })
     helper.registerAllowedMethod('usernamePassword', [Map.class], { m ->
       m.each{ k, v ->

src/test/groovy/GoogleStorageUploadExtStepTests.groovy

@@ -34,14 +34,12 @@ class GoogleStorageUploadExtStepTests extends ApmBasePipelineTest {
   @Test
   void test_windows() throws Exception {
     helper.registerAllowedMethod('isUnix', [], { false })
-    try {
-      script.call(bucket: 'gs://foo', pattern: 'file.txt')
-    } catch(e){
-      //NOOP
-    }
+    helper.registerAllowedMethod('gsutil', [Map.class], { return 'Operation completed over 1 objects.' })
+    def ret = script.call(bucket: 'gs://foo', pattern: 'file.txt')
     printCallStack()
-    assertTrue(assertMethodCallContainsPattern('error', 'gsutil: windows is not supported yet.'))
-    assertJobStatusFailure()
+    assertFalse(assertMethodCallContainsPattern('gsutil', '-a public-read'))
+    assertTrue(assertMethodCallContainsPattern('gsutil', 'file.txt gs://foo'))
+    assertFalse(ret.isEmpty())
   }
 
   @Test

src/test/groovy/GsutilStepTests.groovy

@@ -32,19 +32,6 @@ class GsutilStepTests extends ApmBasePipelineTest {
     script = loadScript('vars/gsutil.groovy')
   }
 
-  @Test
-  void test_windows() throws Exception {
-    helper.registerAllowedMethod('isUnix', [], { false })
-    try {
-      script.call()
-    } catch(e){
-      //NOOP
-    }
-    printCallStack()
-    assertTrue(assertMethodCallContainsPattern('error', 'gsutil: windows is not supported yet.'))
-    assertJobStatusFailure()
-  }
-
   @Test
   void test_without_command() throws Exception {
     try {
@@ -74,6 +61,7 @@ class GsutilStepTests extends ApmBasePipelineTest {
     script.call(command: 'cp', credentialsId: 'foo')
     printCallStack()
     assertTrue(assertMethodCallContainsPattern('withCredentials', ''))
+    assertTrue(assertMethodCallContainsPattern('sh', 'gcloud auth activate-service-account --key-file ${FILE_CREDENTIAL}'))
     assertTrue(assertMethodCallContainsPattern('sh', "gsutil cp"))
     assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL'))
     assertFalse(assertMethodCallContainsPattern('sh', "wget -q -O"))
@@ -102,7 +90,7 @@ class GsutilStepTests extends ApmBasePipelineTest {
     script.call(command: 'cp', credentialsId: 'foo')
     printCallStack()
     assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL'))
-    assertTrue(assertMethodCallContainsPattern('sh', 'wget -q -O'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'wget -q -O gsutil.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-319.0.0-linux-x86_64.tar.gz'))
     assertJobStatusSuccess()
   }
 
@@ -142,4 +130,28 @@ class GsutilStepTests extends ApmBasePipelineTest {
     assertTrue(assertMethodCallContainsPattern('log', 'gsutil: set the gsutilLocation.'))
     assertJobStatusSuccess()
   }
+
+  @Test
+  void test_windows() throws Exception {
+    helper.registerAllowedMethod('isUnix', [], { false })
+    script.call(command: 'cp', credentialsId: 'foo')
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('withCredentials', ''))
+    assertTrue(assertMethodCallContainsPattern('bat', 'gcloud auth activate-service-account --key-file %FILE_CREDENTIAL%'))
+    assertTrue(assertMethodCallContainsPattern('bat', "gsutil cp"))
+    assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL'))
+    assertFalse(assertMethodCallContainsPattern('bat', "wget -q -O"))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void test_without_gh_installed_by_default_with_wget_in_windows() throws Exception {
+    helper.registerAllowedMethod('isInstalled', [Map.class], { m -> return m.tool.equals('wget') })
+    helper.registerAllowedMethod('isUnix', [], { false })
+    script.call(command: 'cp', credentialsId: 'foo')
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL'))
+    assertTrue(assertMethodCallContainsPattern('bat', 'wget -q -O gsutil.zip https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-319.0.0-windows-x86_64.zip'))
+    assertJobStatusSuccess()
+  }
 }

vars/README.md

@@ -1244,6 +1244,32 @@ As long as we got some concurrency issues
 googleStorageUpload(args)
 ```
 
+## googleStorageUploadExt
+Upload the given pattern files to the given bucket.
+
+```
+  // Copy file.txt into the bucket
+  googleStorageUploadExt(pattern: 'file.txt', bucket: 'gs://bucket/folder/', credentialsId: 'foo', sharedPublicly: false)
+
+```
+
+* bucket: The Google Storage bucket format gs://bucket/folder/subfolder/. Mandatory
+* credentialsId: The credentials to access the repo (repo permissions). Optional. Default to `JOB_GCS_CREDENTIALS`
+* pattern: The file to pattern to search and copy. Mandatory.
+* sharedPublicly: Whether to shared those objects publically. Optional. Default false.
+
+## gsutil
+Wrapper to interact with the gsutil command line. It returns the stdout output.
+
+```
+  // Copy file.txt into the bucket
+  gsutil(command: 'cp file.txt gs://bucket/folder/', credentialsId: 'foo' ])
+
+```
+
+* command: The gsutil command to be executed. Mandatory
+* credentialsId: The credentials to access the repo (repo permissions). Mandatory.
+
 ## httpRequest
 Step to make HTTP request and get the result.
 If the return code is >= 400, it would throw an error.

vars/gsutil.groovy

@@ -20,9 +20,6 @@ import groovy.transform.Field
 @Field def gsUtilLocation = ''
 
 def call(Map args = [:]) {
-  if(!isUnix()) {
-    error 'gsutil: windows is not supported yet.'
-  }
   def command = args.containsKey('command') ? args.command : error('gsutil: command argument is required.')
   def credentialsId = args.containsKey('credentialsId') ? args.credentialsId : error('gsutil: credentialsId argument is required.')
 
@@ -38,25 +35,40 @@ def call(Map args = [:]) {
       downloadInstaller(gsUtilLocation)
     }
 
-    withCredentials([file(credentialsId: credentialsId, variable: 'FILE_CREDENTIAL')]) { 
-      sh(label: 'authenticate', script: 'gcloud auth activate-service-account --key-file ${FILE_CREDENTIAL}')
+    withCredentials([file(credentialsId: credentialsId, variable: 'FILE_CREDENTIAL')]) {
+      def credentialsVariable = isUnix() ? '${FILE_CREDENTIAL}' : '%FILE_CREDENTIAL%'
+      cmd(label: 'authenticate', script: 'gcloud auth activate-service-account --key-file ' + credentialsVariable)
     }
-    return sh(label: "gsutil ${command}", script: "gsutil ${command}", returnStdout: true)
+    return cmd(label: "gsutil ${command}", script: "gsutil ${command}", returnStdout: true)
   }
 }
 
 def downloadInstaller(where) {
-  def url = 'https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-319.0.0'
-  url = "${url}-linux-${is64() ? 'x86_64' : 'x86'}.tar.gz"
-  def tarball = 'gsutil.tar.gz'
+  def url = googleCloudSdkURL()
+  def tarball = "gsutil.${isUnix() ? 'tar.gz' : 'zip'}"
   if(isInstalled(tool: 'wget', flag: '--version')) {
     dir(where) {
       retryWithSleep(retries: 3, seconds: 5, backoff: true) {
-        sh(label: 'download gsutil', script: "wget -q -O ${tarball} ${url}")
-        sh(label: 'untar gsutil', script: "tar -xpf ${tarball} --strip-components=1")
+        cmd(label: 'download gsutil', script: "wget -q -O ${tarball} ${url}")
+        uncompress(tarball)
       }
     }
   } else {
     log(level: 'WARN', text: 'gsutil: wget is not available. gsutil will not be installed then.')
   }
-} 
+}
+
+def googleCloudSdkURL() {
+  def url = 'https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-319.0.0'
+  def os = isUnix() ? 'linux' : 'windows'
+  def ext = isUnix() ? 'tar.gz' : 'zip'
+  return "${url}-${os}-${is64() ? 'x86_64' : 'x86'}.${ext}"
+}
+
+def uncompress(tarball) {
+  if (isUnix()) {
+    sh(label: 'untar gsutil', script: "tar -xpf ${tarball} --strip-components=1")
+  } else {
+    unzip(quiet: true, zipFile: tarball)
+  }
+}

vars/gsutil.txt

@@ -8,5 +8,3 @@ Wrapper to interact with the gsutil command line. It returns the stdout output.
 
 * command: The gsutil command to be executed. Mandatory
 * credentialsId: The credentials to access the repo (repo permissions). Mandatory.
-
-_NOTE_: Windows is not supported yet.