feat: stage status cache

src/test/groovy/StageStatusCacheTests.groovy

@@ -0,0 +1,147 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Before
+import org.junit.Test
+import static org.junit.Assert.assertTrue
+import static org.junit.Assert.assertFalse
+
+class StageStatusCacheTests extends ApmBasePipelineTest {
+
+  @Override
+  @Before
+  void setUp() throws Exception {
+    super.setUp()
+    script = loadScript('vars/stageStatusCache.groovy')
+    env.GIT_BASE_COMMIT = "29480a51"
+    env.STAGE_ID = 'fooo'
+    env.FILE_NAME_BASE64 = 'Zm9vbzI5NDgwYTUxCg'
+    env.BUILD_ID = "10"
+    helper.registerAllowedMethod('googleStorageUploadExt', [Map.class], { "OK" })
+    helper.registerAllowedMethod('isUserTrigger', { false })
+    helper.registerAllowedMethod('base64encode', [Map.class], { env.FILE_NAME_BASE64 })
+  }
+
+  @Test
+  void test() throws Exception {
+    helper.registerAllowedMethod('fileExists', [String.class], { false })
+    def isOK = false
+    script.call(id: env.STAGE_ID){
+      isOK = true
+    }
+    printCallStack()
+    assertTrue(isOK)
+    assertTrue(assertMethodCallContainsPattern('base64encode', "${env.STAGE_ID}${env.GIT_BASE_COMMIT}"))
+    assertTrue(assertMethodCallContainsPattern('cmd', 'Download Stage Status'))
+    assertTrue(assertMethodCallContainsPattern('fileExists', "${env.FILE_NAME_BASE64}"))
+    assertTrue(assertMethodCallContainsPattern('writeFile', "file=${env.FILE_NAME_BASE64}"))
+    assertTrue(assertMethodCallContainsPattern('googleStorageUploadExt', "pattern=${env.FILE_NAME_BASE64}"))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void testCache() throws Exception {
+    helper.registerAllowedMethod('fileExists', [String.class], { true })
+    def isOK = true
+    script.call(id: env.STAGE_ID){
+      isOK = false
+    }
+    printCallStack()
+    assertTrue(isOK)
+    assertTrue(assertMethodCallContainsPattern('base64encode', "${env.STAGE_ID}${env.GIT_BASE_COMMIT}"))
+    assertTrue(assertMethodCallContainsPattern('cmd', 'Download Stage Status'))
+    assertTrue(assertMethodCallContainsPattern('fileExists', "${env.FILE_NAME_BASE64}"))
+    assertTrue(assertMethodCallContainsPattern('log', "The stage skiped because it is in the execution cache."))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void testNoCheckCacheOnFirstBuild() throws Exception {
+    helper.registerAllowedMethod('fileExists', [String.class], { true })
+    env.BUILD_ID = "1"
+    def isOK = false
+    script.call(id: env.STAGE_ID){
+      isOK = true
+    }
+    printCallStack()
+    assertTrue(isOK)
+    assertTrue(assertMethodCallContainsPattern('base64encode', "${env.STAGE_ID}${env.GIT_BASE_COMMIT}"))
+    assertFalse(assertMethodCallContainsPattern('cmd', 'Download Stage Status'))
+    assertFalse(assertMethodCallContainsPattern('fileExists', "${env.FILE_NAME_BASE64}"))
+    assertTrue(assertMethodCallContainsPattern('writeFile', "file=${env.FILE_NAME_BASE64}"))
+    assertTrue(assertMethodCallContainsPattern('googleStorageUploadExt', "pattern=${env.FILE_NAME_BASE64}"))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void testNoCheckCacheOnRunAlways() throws Exception {
+    helper.registerAllowedMethod('fileExists', [String.class], { true })
+    def isOK = false
+    script.call(id: env.STAGE_ID, runAlways: true){
+      isOK = true
+    }
+    printCallStack()
+    assertTrue(isOK)
+    assertTrue(assertMethodCallContainsPattern('base64encode', "${env.STAGE_ID}${env.GIT_BASE_COMMIT}"))
+    assertFalse(assertMethodCallContainsPattern('cmd', 'Download Stage Status'))
+    assertFalse(assertMethodCallContainsPattern('fileExists', "${env.FILE_NAME_BASE64}"))
+    assertTrue(assertMethodCallContainsPattern('writeFile', "file=${env.FILE_NAME_BASE64}"))
+    assertTrue(assertMethodCallContainsPattern('googleStorageUploadExt', "pattern=${env.FILE_NAME_BASE64}"))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void testNoCheckCacheOnUserTrigger() throws Exception {
+    helper.registerAllowedMethod('fileExists', [String.class], { true })
+    helper.registerAllowedMethod('isUserTrigger', { true })
+    def isOK = false
+    script.call(id: env.STAGE_ID){
+      isOK = true
+    }
+    printCallStack()
+    assertTrue(isOK)
+    assertTrue(assertMethodCallContainsPattern('base64encode', "${env.STAGE_ID}${env.GIT_BASE_COMMIT}"))
+    assertFalse(assertMethodCallContainsPattern('cmd', 'Download Stage Status'))
+    assertFalse(assertMethodCallContainsPattern('fileExists', "${env.FILE_NAME_BASE64}"))
+    assertTrue(assertMethodCallContainsPattern('writeFile', "file=${env.FILE_NAME_BASE64}"))
+    assertTrue(assertMethodCallContainsPattern('googleStorageUploadExt', "pattern=${env.FILE_NAME_BASE64}"))
+    assertJobStatusSuccess()
+  }
+
+
+  @Test
+  void testParams() throws Exception {
+    helper.registerAllowedMethod('fileExists', [String.class], { false })
+    helper.registerAllowedMethod('isUserTrigger', { true })
+    env.FILE_NAME_BASE64 = 'Zm9vb2Zvb1NIQQo'
+    def isOK = false
+    script.call(id: env.STAGE_ID,
+      bucket: 'bucketFoo',
+      credentialsId: 'fooCredentials',
+      sha: 'fooSHA'){
+      isOK = true
+    }
+    printCallStack()
+    assertTrue(isOK)
+    assertTrue(assertMethodCallContainsPattern('base64encode', "fooofooSHA"))
+    assertTrue(assertMethodCallContainsPattern('writeFile', "file=${env.FILE_NAME_BASE64}"))
+    assertTrue(assertMethodCallContainsPattern('googleStorageUploadExt', "pattern=${env.FILE_NAME_BASE64}"))
+    assertTrue(assertMethodCallContainsPattern('googleStorageUploadExt', "bucket=gs://bucketFoo/ci/cache/"))
+    assertTrue(assertMethodCallContainsPattern('googleStorageUploadExt', "credentialsId=fooCredentials"))
+    assertJobStatusSuccess()
+  }
+}

vars/README.md

@@ -2350,6 +2350,42 @@ setupAPMGitEmail(global: true)
   stackVersions.edge(snapshot: true) // '8.0.0-SNAPSHOT'
 ```
 
+## stageStatusCache
+Stage status cache allow to save and restore the status of a stage for a particular commit.
+This allow to skip stages when we know that we executed that stage for that commit.
+To do that the step save a file based on `stageSHA|base64` on a GCP bucket,
+this status is checked and execute the body if there is not stage status file
+for the stage and the commit we are building.
+User triggered builds will execute all stages always.
+If the stage success the status is save in a file.
+It uses `GIT_BASE_COMMIT` as a commit SHA, because is a known real commit SHA,
+because of that merges with target branch will skip stages on changes only on target branch.
+
+```
+pipeline {
+  agent any
+  stages {
+    stage('myStage') {
+      steps {
+        deleteDir()
+        stageStatusCache(id: 'myStage',
+          bucket: 'myBucket',
+          credentialsId: 'my-credentials',
+          sha: getGitCommitSha()
+        ){
+          echo "My code"
+        }
+      }
+    }
+  }
+}
+```
+
+* *id:* Unique stage name. Mandatory
+* *bucket:* bucket name. Default 'beats-ci-temp'
+* *credentialsId:* credentials file, with the GCP credentials JSON file. Default  'beats-ci-gcs-plugin-file-credentials'
+* *sha:* Commit SHA used for the stage ID. Default: env.GIT_BASE_COMMIT
+
 ## stashV2
 Stash the current location, for such it compresses the current path and
 upload it to Google Storage.

vars/stageStatusCache.groovy

@@ -0,0 +1,72 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/**
+  Execute the body if there is not stage status file for the stage and the commit we are building.
+  User triggered builds will execute all stages always.
+  If the stage success the status is save in a file.
+*/
+def call(Map args, Closure body){
+  if(!args.containsKey('bucket')){
+    args['bucket'] = 'beats-ci-temp'
+  }
+  if(!args.containsKey('credentialsId')){
+    args['credentialsId'] = 'beats-ci-gcs-plugin-file-credentials'
+  }
+  if(!args.containsKey('sha')){
+    args['sha'] = "${env.GIT_BASE_COMMIT}"
+  }
+  if(env.BUILD_ID == "1" || isUserTrigger() || args.get('runAlways', false) || readStageStatus(args) == false){
+    body()
+    saveStageStatus(args)
+  } else {
+    log(level: 'INFO', text: "The stage skiped because it is in the execution cache.")
+  }
+}
+
+/**
+  Save the status file of the stage.
+*/
+def saveStageStatus(Map args){
+  def statusFileName = stageStatusId(args)
+  writeFile(file: statusFileName, text: "OK")
+  googleStorageUploadExt(bucket: "gs://${args.bucket}/ci/cache/",
+    credentialsId: "${args.credentialsId}",
+    pattern: "${statusFileName}",
+    sharedPublicly: true)
+}
+
+/**
+  Read the status file of the stage if it exists.
+*/
+def readStageStatus(Map args){
+  def statusFileName = stageStatusId(args)
+  try {
+    cmd(label: 'Download Stage Status',
+      script: "curl -sSf -O https://storage.googleapis.com/${args.bucket}/ci/cache/${statusFileName}",
+      returnStatus: true)
+  } finally {
+    return fileExists("${statusFileName}")
+  }
+}
+
+/**
+  generate an unique ID for the stage and commit.
+*/
+def stageStatusId(Map args){
+  return base64encode(text: "${args.id}${args.sha}", encoding: "UTF-8", padding: false)
+}

vars/stageStatusCache.txt

@@ -0,0 +1,34 @@
+Stage status cache allow to save and restore the status of a stage for a particular commit.
+This allow to skip stages when we know that we executed that stage for that commit.
+To do that the step save a file based on `stageSHA|base64` on a GCP bucket,
+this status is checked and execute the body if there is not stage status file
+for the stage and the commit we are building.
+User triggered builds will execute all stages always.
+If the stage success the status is save in a file.
+It uses `GIT_BASE_COMMIT` as a commit SHA, because is a known real commit SHA,
+because of that merges with target branch will skip stages on changes only on target branch.
+
+```
+pipeline {
+  agent any
+  stages {
+    stage('myStage') {
+      steps {
+        deleteDir()
+        stageStatusCache(id: 'myStage',
+          bucket: 'myBucket',
+          credentialsId: 'my-credentials',
+          sha: getGitCommitSha()
+        ){
+          echo "My code"
+        }
+      }
+    }
+  }
+}
+```
+
+* *id:* Unique stage name. Mandatory
+* *bucket:* bucket name. Default 'beats-ci-temp'
+* *credentialsId:* credentials file, with the GCP credentials JSON file. Default  'beats-ci-gcs-plugin-file-credentials'
+* *sha:* Commit SHA used for the stage ID. Default: env.GIT_BASE_COMMIT