diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 238e3fc81..902497625 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -117,13 +117,25 @@ pipeline { } } stage('Google Storage') { + steps { + withGithubNotify(context: 'Google Storage', description: 'Google Storage step') { + dir("${BASE_DIR}"){ + googleStorageUploadExt(bucket: "gs://${env.JOB_GCS_BUCKET}/${env.REPO}/commits/${env.GIT_BASE_COMMIT}", pattern: 'src/**/*.gdsl', sharedPublicly: false) + } + } + } + } + stage('Google Storage Windows') { + agent { label 'windows-2019-immutable' } when { - branch 'master' // TODO: to be removed as soon as the step is available in the master branch + branch 'master' } steps { - withGithubNotify(context: 'Google Storage', description: 'Google Storage step') { + withGithubNotify(context: 'Google Storage Windows', description: 'Google Storage step') { + deleteDir() + unstash 'source' dir("${BASE_DIR}"){ - googleStorageUploadExt(bucket: "gs://${env.JOB_GCS_BUCKET}/${env.REPO}/commits/${env.GIT_BASE_COMMIT}", pattern: 'target/**/*.xml', sharedPublicly: false) + googleStorageUploadExt(bucket: "gs://${env.JOB_GCS_BUCKET}/${env.REPO}/commits/${env.GIT_BASE_COMMIT}", pattern: 'resources/**/*.json', sharedPublicly: false) } } } @@ -135,9 +147,15 @@ pipeline { validateDeclarativePipeline '.ci/Jenkinsfile' validateDeclarativePipeline '.ci/buildBeatsDockerImages.groovy' validateDeclarativePipeline '.ci/buildDockerImages.groovy' + validateDeclarativePipeline '.ci/chatops.groovy' + validateDeclarativePipeline '.ci/buildDockerImages.groovy' validateDeclarativePipeline '.ci/dockerImagesOpbeans.groovy' + validateDeclarativePipeline '.ci/jobdsl-generator.groovy' + validateDeclarativePipeline '.ci/licenseScan.groovy' + validateDeclarativePipeline '.ci/licenseScanGeneral.groovy' validateDeclarativePipeline '.ci/schedule-daily.groovy' validateDeclarativePipeline '.ci/schedule-weekly.groovy' + validateDeclarativePipeline '.ci/validateWorkersBeatsCi.groovy' validateDeclarativePipeline 'resources/JenkinsfileTemplate.groovy' } } diff --git a/src/test/groovy/ApmBasePipelineTest.groovy b/src/test/groovy/ApmBasePipelineTest.groovy index fe1419d69..f9659a3a2 100644 --- a/src/test/groovy/ApmBasePipelineTest.groovy +++ b/src/test/groovy/ApmBasePipelineTest.groovy @@ -334,6 +334,7 @@ class ApmBasePipelineTest extends DeclarativePipelineTest { helper.registerAllowedMethod('string', [Map.class], { m -> return m }) helper.registerAllowedMethod('timeout', [Integer.class, Closure.class], null) helper.registerAllowedMethod('unstash', [String.class], null) + helper.registerAllowedMethod('unzip', [Map.class], null) helper.registerAllowedMethod('upstreamDevelopers', { "OK" }) helper.registerAllowedMethod('usernamePassword', [Map.class], { m -> m.each{ k, v -> diff --git a/src/test/groovy/GoogleStorageUploadExtStepTests.groovy b/src/test/groovy/GoogleStorageUploadExtStepTests.groovy index 6dc2a077b..427725575 100644 --- a/src/test/groovy/GoogleStorageUploadExtStepTests.groovy +++ b/src/test/groovy/GoogleStorageUploadExtStepTests.groovy @@ -34,14 +34,12 @@ class GoogleStorageUploadExtStepTests extends ApmBasePipelineTest { @Test void test_windows() throws Exception { helper.registerAllowedMethod('isUnix', [], { false }) - try { - script.call(bucket: 'gs://foo', pattern: 'file.txt') - } catch(e){ - //NOOP - } + helper.registerAllowedMethod('gsutil', [Map.class], { return 'Operation completed over 1 objects.' }) + def ret = script.call(bucket: 'gs://foo', pattern: 'file.txt') printCallStack() - assertTrue(assertMethodCallContainsPattern('error', 'gsutil: windows is not supported yet.')) - assertJobStatusFailure() + assertFalse(assertMethodCallContainsPattern('gsutil', '-a public-read')) + assertTrue(assertMethodCallContainsPattern('gsutil', 'file.txt gs://foo')) + assertFalse(ret.isEmpty()) } @Test diff --git a/src/test/groovy/GsutilStepTests.groovy b/src/test/groovy/GsutilStepTests.groovy index c495dd605..c07033891 100644 --- a/src/test/groovy/GsutilStepTests.groovy +++ b/src/test/groovy/GsutilStepTests.groovy @@ -32,19 +32,6 @@ class GsutilStepTests extends ApmBasePipelineTest { script = loadScript('vars/gsutil.groovy') } - @Test - void test_windows() throws Exception { - helper.registerAllowedMethod('isUnix', [], { false }) - try { - script.call() - } catch(e){ - //NOOP - } - printCallStack() - assertTrue(assertMethodCallContainsPattern('error', 'gsutil: windows is not supported yet.')) - assertJobStatusFailure() - } - @Test void test_without_command() throws Exception { try { @@ -74,6 +61,7 @@ class GsutilStepTests extends ApmBasePipelineTest { script.call(command: 'cp', credentialsId: 'foo') printCallStack() assertTrue(assertMethodCallContainsPattern('withCredentials', '')) + assertTrue(assertMethodCallContainsPattern('sh', 'gcloud auth activate-service-account --key-file ${FILE_CREDENTIAL}')) assertTrue(assertMethodCallContainsPattern('sh', "gsutil cp")) assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL')) assertFalse(assertMethodCallContainsPattern('sh', "wget -q -O")) @@ -102,7 +90,7 @@ class GsutilStepTests extends ApmBasePipelineTest { script.call(command: 'cp', credentialsId: 'foo') printCallStack() assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL')) - assertTrue(assertMethodCallContainsPattern('sh', 'wget -q -O')) + assertTrue(assertMethodCallContainsPattern('sh', 'wget -q -O gsutil.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-319.0.0-linux-x86_64.tar.gz')) assertJobStatusSuccess() } @@ -142,4 +130,28 @@ class GsutilStepTests extends ApmBasePipelineTest { assertTrue(assertMethodCallContainsPattern('log', 'gsutil: set the gsutilLocation.')) assertJobStatusSuccess() } + + @Test + void test_windows() throws Exception { + helper.registerAllowedMethod('isUnix', [], { false }) + script.call(command: 'cp', credentialsId: 'foo') + printCallStack() + assertTrue(assertMethodCallContainsPattern('withCredentials', '')) + assertTrue(assertMethodCallContainsPattern('bat', 'gcloud auth activate-service-account --key-file %FILE_CREDENTIAL%')) + assertTrue(assertMethodCallContainsPattern('bat', "gsutil cp")) + assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL')) + assertFalse(assertMethodCallContainsPattern('bat', "wget -q -O")) + assertJobStatusSuccess() + } + + @Test + void test_without_gh_installed_by_default_with_wget_in_windows() throws Exception { + helper.registerAllowedMethod('isInstalled', [Map.class], { m -> return m.tool.equals('wget') }) + helper.registerAllowedMethod('isUnix', [], { false }) + script.call(command: 'cp', credentialsId: 'foo') + printCallStack() + assertTrue(assertMethodCallContainsPattern('withEnv', 'PATH+GSUTIL')) + assertTrue(assertMethodCallContainsPattern('bat', 'wget -q -O gsutil.zip https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-319.0.0-windows-x86_64.zip')) + assertJobStatusSuccess() + } } diff --git a/vars/README.md b/vars/README.md index 3103c8180..74aa670fc 100644 --- a/vars/README.md +++ b/vars/README.md @@ -1254,6 +1254,32 @@ As long as we got some concurrency issues googleStorageUpload(args) ``` +## googleStorageUploadExt +Upload the given pattern files to the given bucket. + +``` + // Copy file.txt into the bucket + googleStorageUploadExt(pattern: 'file.txt', bucket: 'gs://bucket/folder/', credentialsId: 'foo', sharedPublicly: false) + +``` + +* bucket: The Google Storage bucket format gs://bucket/folder/subfolder/. Mandatory +* credentialsId: The credentials to access the repo (repo permissions). Optional. Default to `JOB_GCS_CREDENTIALS` +* pattern: The file to pattern to search and copy. Mandatory. +* sharedPublicly: Whether to shared those objects publically. Optional. Default false. + +## gsutil +Wrapper to interact with the gsutil command line. It returns the stdout output. + +``` + // Copy file.txt into the bucket + gsutil(command: 'cp file.txt gs://bucket/folder/', credentialsId: 'foo' ]) + +``` + +* command: The gsutil command to be executed. Mandatory +* credentialsId: The credentials to access the repo (repo permissions). Mandatory. + ## httpRequest Step to make HTTP request and get the result. If the return code is >= 400, it would throw an error. diff --git a/vars/gsutil.groovy b/vars/gsutil.groovy index a2d0ffea0..3a66203e7 100644 --- a/vars/gsutil.groovy +++ b/vars/gsutil.groovy @@ -20,9 +20,6 @@ import groovy.transform.Field @Field def gsUtilLocation = '' def call(Map args = [:]) { - if(!isUnix()) { - error 'gsutil: windows is not supported yet.' - } def command = args.containsKey('command') ? args.command : error('gsutil: command argument is required.') def credentialsId = args.containsKey('credentialsId') ? args.credentialsId : error('gsutil: credentialsId argument is required.') @@ -38,25 +35,40 @@ def call(Map args = [:]) { downloadInstaller(gsUtilLocation) } - withCredentials([file(credentialsId: credentialsId, variable: 'FILE_CREDENTIAL')]) { - sh(label: 'authenticate', script: 'gcloud auth activate-service-account --key-file ${FILE_CREDENTIAL}') + withCredentials([file(credentialsId: credentialsId, variable: 'FILE_CREDENTIAL')]) { + def credentialsVariable = isUnix() ? '${FILE_CREDENTIAL}' : '%FILE_CREDENTIAL%' + cmd(label: 'authenticate', script: 'gcloud auth activate-service-account --key-file ' + credentialsVariable) } - return sh(label: "gsutil ${command}", script: "gsutil ${command}", returnStdout: true) + return cmd(label: "gsutil ${command}", script: "gsutil ${command}", returnStdout: true) } } def downloadInstaller(where) { - def url = 'https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-319.0.0' - url = "${url}-linux-${is64() ? 'x86_64' : 'x86'}.tar.gz" - def tarball = 'gsutil.tar.gz' + def url = googleCloudSdkURL() + def tarball = "gsutil.${isUnix() ? 'tar.gz' : 'zip'}" if(isInstalled(tool: 'wget', flag: '--version')) { dir(where) { retryWithSleep(retries: 3, seconds: 5, backoff: true) { - sh(label: 'download gsutil', script: "wget -q -O ${tarball} ${url}") - sh(label: 'untar gsutil', script: "tar -xpf ${tarball} --strip-components=1") + cmd(label: 'download gsutil', script: "wget -q -O ${tarball} ${url}") + uncompress(tarball) } } } else { log(level: 'WARN', text: 'gsutil: wget is not available. gsutil will not be installed then.') } -} +} + +def googleCloudSdkURL() { + def url = 'https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-319.0.0' + def os = isUnix() ? 'linux' : 'windows' + def ext = isUnix() ? 'tar.gz' : 'zip' + return "${url}-${os}-${is64() ? 'x86_64' : 'x86'}.${ext}" +} + +def uncompress(tarball) { + if (isUnix()) { + sh(label: 'untar gsutil', script: "tar -xpf ${tarball} --strip-components=1") + } else { + unzip(quiet: true, zipFile: tarball) + } +} diff --git a/vars/gsutil.txt b/vars/gsutil.txt index 05621a8e0..88b723924 100644 --- a/vars/gsutil.txt +++ b/vars/gsutil.txt @@ -8,5 +8,3 @@ Wrapper to interact with the gsutil command line. It returns the stdout output. * command: The gsutil command to be executed. Mandatory * credentialsId: The credentials to access the repo (repo permissions). Mandatory. - -_NOTE_: Windows is not supported yet.