[CD] Add COMMIT_ID param to release job (#16202)

* Removes unnecessary parameter

* Adds revision parameter to release job

* Update documentation

* Changes RELEASE_JOB_TYPE to string parameter

* Adds variant and release job type checks

* Adds release job update to CD pipeline

* Disable concurrent builds for CD pipeline

* Updates documentation with new release job mechanics

cd/Jenkinsfile_cd_pipeline

@@ -27,6 +27,12 @@ pipeline {
   agent {
     label 'restricted-utility'
   }
+  options {
+    // Because each pass of the CD pipeline
+    // updates Jenkins' state of the release job
+    // to avoid crazy issues, we don't allow concurrent builds.
+    disableConcurrentBuilds()  
+  }
 
   parameters {
     // Release parameters
@@ -39,6 +45,9 @@ pipeline {
       steps {
         script {
           cd_utils = load('cd/Jenkinsfile_utils.groovy')
+
+          // Update release job state in Jenkins
+          cd_utils.update_release_job_state()
         }
       }
     }

cd/Jenkinsfile_release_job

@@ -37,15 +37,20 @@ pipeline {
   parameters {
     // Release parameters
     string(defaultValue: "Generic release job", description: "Optional Job name", name: "RELEASE_JOB_NAME")
-    choice(choices: ["mxnet_lib/static", "mxnet_lib/dynamic"], description: "Pipeline to build", name: "RELEASE_JOB_TYPE")
+    string(defaultValue: "master", description: "Git Commit to Build", name: "COMMIT_ID")
+
+    // Using string instead of choice parameter to keep the changes to the parameters minimal to avoid
+    // any disruption caused by different COMMIT_ID values chaning the job parameter configuration on 
+    // Jenkins.
+    string(defaultValue: "mxnet_lib/static", description: "Pipeline to build", name: "RELEASE_JOB_TYPE")
     string(defaultValue: "cpu,mkl,cu90,cu90mkl,cu92,cu92mkl,cu100,cu100mkl,cu101,cu101mkl", description: "Comma separated list of variants", name: "MXNET_VARIANTS")
     booleanParam(defaultValue: false, description: 'Whether this is a release build or not', name: "RELEASE_BUILD")
   }
 
   stages {
     stage("Init") {
       steps {
-        script{
+        script {
           cd_utils = load('cd/Jenkinsfile_utils.groovy')
           ci_utils = load('ci/Jenkinsfile_utils.groovy')
           ci_utils.assign_node_labels(
@@ -57,23 +62,52 @@ pipeline {
             windows_gpu: 'restricted-mxnetwindows-gpu'
           )
 
-          echo """\
+          // Skip Jenkins state update jobs
+          if (env.RELEASE_JOB_TYPE == cd_utils.STATE_UPDATE) {
+            echo """\
+            |Job Type: ${env.RELEASE_JOB_TYPE}
+            |Commit Id: ${env.GIT_COMMIT}""".stripMargin()
+          } else {
+            echo """\
             |Job Name: ${env.RELEASE_JOB_NAME}
             |Job Type: ${env.RELEASE_JOB_TYPE}
             |Release Build: ${params.RELEASE_BUILD}
             |Commit Id: ${env.GIT_COMMIT}
             |Branch: ${env.GIT_BRANCH}
             |Variants: ${env.MXNET_VARIANTS}""".stripMargin()
+          }
         }
       }
     }
     stage("Release Job") {
       steps {
         script {
+          // Skip builds for state update job
+          if (env.RELEASE_JOB_TYPE == cd_utils.STATE_UPDATE) {
+            currentBuild.result = "SUCCESS"
+            return
+          }
+
+          // Add new job types here
+          def valid_job_types = [
+            "mxnet_lib/static",
+            "mxnet_lib/dynamic", 
+            "python/pypi"
+          ]
 
           // Convert mxnet variants to a list
-          def mxnet_variants = params.MXNET_VARIANTS.split(',').inject([]) { list, item ->
+          def mxnet_variants = params.MXNET_VARIANTS.trim().split(',').inject([]) { list, item ->
             list << item.trim()
+          }.findAll { item -> ! (item == null || item.isEmpty()) }
+
+          // Exit successfully if there are no variants to build
+          if (mxnet_variants.size() == 0) {
+            error "No variants to build..."
+          }
+
+          // Only execute from allowed release job types
+          if (! (valid_job_types.contains(params.RELEASE_JOB_TYPE))) {
+            error "Unknown release job type '${params.RELEASE_JOB_TYPE}'"
           }
 
           // Load script for the supplied job type

cd/Jenkinsfile_utils.groovy

@@ -23,14 +23,18 @@
 // 'Jenkins_pipeline.groovy' file and has the pipeline definition for the 
 // artifact (docker image, binary, pypi or maven package, etc.) that should
 // be published.
+
+STATE_UPDATE="State Update"
+
 def trigger_release_job(job_name, job_type, mxnet_variants) {
   def run = build(
     job: env.CD_RELEASE_JOB_NAME, 
     parameters: [
-      string(name: 'RELEASE_JOB_NAME', value: "${job_name}"),
-      string(name: 'RELEASE_JOB_TYPE', value: "${job_type}"),
-      string(name: 'MXNET_VARIANTS', value: "${mxnet_variants}"),
-      booleanParam(name: 'RELEASE_BUILD', value: "${env.RELEASE_BUILD}")
+      string(name: "RELEASE_JOB_NAME", value: "${job_name}"),
+      string(name: "RELEASE_JOB_TYPE", value: "${job_type}"),
+      string(name: "MXNET_VARIANTS", value: "${mxnet_variants}"),
+      booleanParam(name: "RELEASE_BUILD", value: "${env.RELEASE_BUILD}"),
+      string(name: "COMMIT_ID", value: "${env.GIT_COMMIT}")
     ],
     // If propagate is true, any result other than successful will
     // mark this call as failure (inc. unstable).
@@ -55,6 +59,23 @@ def trigger_release_job(job_name, job_type, mxnet_variants) {
   }
 }
 
+
+// This triggers a downstream release job with no
+// variants and not job type. This will update
+// the configuration of the release job in jenkins
+// to the configuration of release job as defined in the
+// Jenkinsfile _release_job for env.GIT_COMMIT revision
+def update_release_job_state() {
+  build(
+    job: env.CD_RELEASE_JOB_NAME, 
+    parameters: [
+      string(name: "RELEASE_JOB_TYPE", value: STATE_UPDATE),
+
+      // Should be set to the current git commit
+      string(name: "COMMIT_ID", value: "${env.GIT_COMMIT}")
+    ])
+}
+
 // Wraps variant pipeline with error catching and
 // job status setting code
 // If there's an error in one of the pipelines, set status to UNSTABLE

cd/README.md

@@ -61,9 +61,18 @@ The [release job](Jenkinsfile_release_job) takes five parameters:
  * **MXNET_VARIANTS**: A comma separated list of variants to build. Defaults to *all* variants.
  * **RELEASE\_JOB\_NAME**: A name for this release job (Optional). Defaults to "Generic release job". It is used for debug output purposes.
  * **RELEASE\_JOB\_TYPE**: Defines the release pipeline you want to execute.
+ * **COMMIT_ID**: The commit id to build
 
 The release job executes, in parallel, the release pipeline for each of the variants (**MXNET_VARIANTS**) for the job type (**RELEASE\_JOB\_TYPE**). The job type the path to a directory (relative to the `cd` directory) that includes a `Jenkins_pipeline.groovy` file ([e.g.](mxnet_lib/static/Jenkins_pipeline.groovy)).
 
+NOTE: The **COMMIT_ID** is a little tricky and we must be very careful with it. It is necessary to ensure that the same commit is built through out the pipeline, but at the same time, it has the potential to change the current state of the release job configuration - specifically the parameter configuration. Any changes to this configuration will require a "dry-run" of the release job to ensure Jenkins has the current (master) version. This is acceptable as there will be few changes to the parameter configuration for the job, if any at all. But, it's something to keep in mind.
+
+To avoid potential issues as much as possible, the CD pipeline executes this "dry run" and ensures that Jenkins' state of the release job matches what is defined for the release job in the specified COMMIT_ID. This is done by setting the **RELEASE_JOB_TYPE** to *Status Update*.
+
+It should be noted that the 'Pipeline' section of the configuration should use the *$COMMIT_ID* parameter as the specifier and 'lightweight checkout' unchecked. For example:
+
+![job setup example](img/job_setup.png)
+
 ### Release Pipelines: Jenkins_pipeline.groovy
 
 This file defines the release pipeline for a particular release channel. It defines a function `get_pipeline(mxnet_variant)`, which returns a closure with the pipeline to be executed. For instance:

ci/Jenkinsfile_utils.groovy

@@ -18,7 +18,7 @@
 // under the License.
 
 // initialize source codes
-def init_git(git_sha = '') {
+def init_git() {
   deleteDir()
   retry(5) {
     try {
@@ -80,8 +80,8 @@ return 0
 }
 
 // unpack libraries saved before
-def unpack_and_init(name, libs, include_gcov_data = false, git_sha = '') {
-  init_git(git_sha)
+def unpack_and_init(name, libs, include_gcov_data = false) {
+  init_git()
   unstash name
   sh returnStatus: true, script: """
 set +e