add website artifacts pipeline (#19397)

Co-authored-by: Wei Chu <[email protected]>

ci/docker/runtime_functions.sh

@@ -1920,6 +1920,27 @@ build_docs_beta() {
     popd
 }
 
+push_docs() {
+    folder_name=$1
+    set -ex
+    pip3 install --user awscli
+    export PATH=~/.local/bin:$PATH
+    pushd docs/_build
+    wget https://mxnet-website-static-artifacts.s3.us-east-2.amazonaws.com/versions.zip && unzip versions.zip && rm versions.zip
+    mkdir $folder_name && tar -xzf full_website.tgz -C $folder_name --strip-components 1
+    # check if folder_name already exists in versions
+    pushd versions
+    if [ -d "$folder_name" ]; then
+        echo "Folder $folder_name already exists in versions. Please double check the FOLDER_NAME variable in Jenkens pipeline"
+        exit 1
+    fi
+    popd
+    mv $folder_name versions
+    zip -r9 versions.zip versions/.
+    aws s3 cp versions.zip s3://mxnet-website-static-artifacts --acl public-read
+    popd
+}
+
 create_repo() {
    repo_folder=$1
    mxnet_url=$2

ci/jenkins/Jenkins_steps.groovy

@@ -1587,6 +1587,55 @@ def docs_jekyll() {
     }]
 }
 
+// This is for building the full website
+// Assumes you have run all of the docs generation functions
+def docs_full_website() {
+    return ['Build artifacts full_website.tgz': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/docs') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.init_git()
+
+            unstash 'jekyll-artifacts'
+            unstash 'c-artifacts'
+            unstash 'python-artifacts'
+            unstash 'r-artifacts'
+            unstash 'julia-artifacts'
+            unstash 'scala-artifacts'
+            unstash 'java-artifacts'
+            unstash 'clojure-artifacts'
+
+            utils.docker_run('ubuntu_cpu_jekyll', 'build_docs', false)
+            utils.pack_lib('full_website', 'docs/_build/full_website.tgz', false)
+          }
+        }
+      }
+    }]
+}
+
+// This is for uploading website artifacts to S3 bucket
+// Assumes you have run docs_full_website function
+def docs_upload_s3() {
+    return ['Upload artifacts to s3 bucket': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/docs') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            if(env.FOLDER_NAME) {
+              utils.unpack_and_init('full_website', 'docs/_build/full_website.tgz')
+
+              utils.docker_run('ubuntu_cpu', "push_docs ${env.FOLDER_NAME}", false)
+
+              archiveArtifacts 'docs/_build/versions.zip'
+            } else {
+              sh 'echo Can not find website version for release. Please specify env var FOLDER_NAME in Jenkins pipeline'
+              sh 'exit 1'
+            }
+
+          }
+        }
+      }
+    }]
+}
 
 // This is for publishing the full website
 // Assumes you have run all of the docs generation functions

ci/jenkins/Jenkinsfile_website_version_artifacts

@@ -0,0 +1,67 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline to build website artifacit and upload to private s3 bucket
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 180
+
+node('restricted-utility') {
+  // Loading the utilities requires a node context unfortunately
+  checkout scm
+  utils = load('ci/Jenkinsfile_utils.groovy')
+  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
+}
+
+utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')
+
+utils.main_wrapper(
+core_logic: {
+  utils.parallel_stage('Build', [
+    custom_steps.compile_unix_lite()
+  ])
+
+  utils.parallel_stage('Build Api Docs', [
+    custom_steps.docs_jekyll(),
+    custom_steps.docs_c(),
+    custom_steps.docs_python(),
+    custom_steps.docs_julia(),
+    custom_steps.docs_r(),
+    custom_steps.docs_scala(),
+    custom_steps.docs_java(),
+    custom_steps.docs_clojure()
+  ])
+
+  utils.parallel_stage('Build Full Website', [
+    custom_steps.docs_full_website()
+  ])
+
+  utils.parallel_stage('Upload Docs', [
+    custom_steps.docs_upload_s3()
+  ])
+}
+,
+failure_handler: {
+  // Only send email if master or release branches failed
+  if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) {
+    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+  }
+}
+)