From 2ecb0795b5bb91e85221ae963c6bdb88d2e81a1a Mon Sep 17 00:00:00 2001
From: Per Goncalves da Silva <perdasilva@gmail.com>
Date: Thu, 23 May 2019 12:48:44 +0200
Subject: [PATCH 1/5] Adds artifact restoration and generic pipeline to cd
 utils

---
 cd/Jenkinsfile_utils.groovy                   | 72 +++++++++++++++++++
 .../mxnet_lib/dynamic/Jenkins_pipeline.groovy | 57 +++++++++++++++
 2 files changed, 129 insertions(+)
 create mode 100644 ci/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy

diff --git a/cd/Jenkinsfile_utils.groovy b/cd/Jenkinsfile_utils.groovy
index da17c19b562c..370e746cb954 100644
--- a/cd/Jenkinsfile_utils.groovy
+++ b/cd/Jenkinsfile_utils.groovy
@@ -119,4 +119,76 @@ def push_artifact(libmxnet_path, variant, libtype, license_paths = '', dependenc
   sh "./cd/utils/artifact_repository.py --push --verbose --libtype ${libtype} --variant ${variant} --libmxnet ${libmxnet_path} --licenses ${license_paths} --dependencies ${dependency_paths}"
 }
 
+// pull artifact from repository
+def pull_artifact(variant, libtype, destination = '') {
+  sh "./ci/cd/utils/artifact_repository.py --pull --verbose --libtype ${libtype} --variant ${variant} --destination ${destination}"
+}
+
+// pulls artifact from repository and places files in the appropriate directories
+def restore_artifact(variant, libtype) {
+  pull_artifact(variant, libtype, 'mxnet_artifact')
+  // move libraries to lib directory
+
+  dir('lib') {
+    sh "mv ../mxnet_artifact/libmxnet.so ."
+    if (fileExists('../mxnet_artifact/dependencies')) {
+      sh """find "../mxnet_artifact/dependencies" -type f -name "*.so*" -exec mv {} . \\;"""
+      sh "ls ."
+    }
+  }
+
+  dir('cd_misc') {
+    if (fileExists('../mxnet_artifact/dependencies')) {
+      // All library files (*.so*) should have be moved
+      // to the lib directory. If anything is left, it will be
+      // other supporting files (header files, etc.)
+      sh """find "../mxnet_artifact/dependencies" -type f -exec mv {} . \\;"""
+      sh "ls ."
+    }
+  }
+
+  dir('licenses') {
+    if (fileExists('../mxnet_artifact/licenses')) {
+      sh """find "../mxnet_artifact/licenses" -type f -exec mv {} . \\;"""
+      sh "ls ."
+    }
+  }
+
+  dir('mxnet_artifact') {
+    deleteDir()
+  }
+}
+
+// A generic pipeline that can be used by *most* CD jobs
+// It can use used when implementing the pipeline steps in the Jenkins_steps.groovy
+// script for the particular delivery channel. However, it should also implemente the
+// build, test, and push steps.
+// NOTE: Be mindful of the expected time that a step should take. If it will take a long time,
+// and it can be done in a CPU node, do it in a CPU node. We should avoid using GPU instances unless
+// we *have* to.
+// However, if it is only packaging libmxnet and that doesn't take long. The the pipeline can 
+// just run on a single node. As is done bellow.
+// For examples of multi-node CD pipelines, see the the binary_release/static and binary_release/dynamic
+// pipeline.
+def generic_pipeline(mxnet_variant, custom_steps, node_type = "restricted-mxnetlinux-cpu") {
+  return {
+    node(node_type) {
+      stage("${mxnet_variant}") {
+
+        stage('Build') {
+          custom_steps.build(mxnet_variant)
+        }
+
+        stage('Test') {
+          custom_steps.test(mxnet_variant)
+        }
+
+        stage('Push') {
+          custom_steps.push(mxnet_variant)
+        }
+      }
+    }
+  }
+}
+
 return this
diff --git a/ci/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy b/ci/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy
new file mode 100644
index 000000000000..0e70e4d9dbe5
--- /dev/null
+++ b/ci/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy
@@ -0,0 +1,57 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// NOTE: ci_utils is loaded by the originating Jenkins job, e.g. jenkins/Jenkinsfile_release_job
+
+// libmxnet location
+libmxnet = 'lib/libmxnet.so'
+
+// licenses
+licenses = 'licenses/*'
+
+// libmxnet dependencies
+mx_deps = ''
+mx_mkldnn_deps = 'lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so'
+
+// library type
+// either static or dynamic - depending on how it links to its dependencies
+libtype = 'dynamic'
+
+libmxnet_pipeline = load('ci/cd/mxnet_lib/mxnet_lib_pipeline.groovy')
+
+// Builds the dynamic binary for the specified mxnet variant
+def build(mxnet_variant) {
+  node(NODE_LINUX_CPU) {
+    ws("workspace/mxnet_${libtype}/${mxnet_variant}/${env.BUILD_NUMBER}") {
+      def image = libmxnet_pipeline.get_environment(mxnet_variant)
+      ci_utils.init_git()
+      ci_utils.docker_run(image, "build_dynamic_libmxnet ${mxnet_variant}", false)
+      ci_utils.pack_lib("mxnet_${mxnet_variant}", libmxnet_pipeline.get_stash(mxnet_variant))
+    }
+  }
+}
+
+def get_pipeline(mxnet_variant) {
+  return libmxnet_pipeline.get_pipeline(mxnet_variant, this.&build)
+}
+
+return this

From 62ca34c7ec4435bb33acba0879123207002372a1 Mon Sep 17 00:00:00 2001
From: Per Goncalves da Silva <perdasilva@gmail.com>
Date: Fri, 24 May 2019 09:02:39 +0200
Subject: [PATCH 2/5] Adds PyPI release pipeline

---
 cd/Jenkinsfile_cd_pipeline                    |  4 +
 cd/Jenkinsfile_utils.groovy                   | 11 +--
 cd/README.md                                  |  8 ++
 cd/python/pypi/Jenkins_pipeline.groovy        | 68 +++++++++++++++
 cd/python/pypi/README.md                      | 43 +++++++++
 cd/python/pypi/pypi_package.sh                | 59 +++++++++++++
 cd/python/pypi/pypi_publish.py                | 87 +++++++++++++++++++
 .../mxnet_lib/dynamic/Jenkins_pipeline.groovy | 57 ------------
 ci/docker/runtime_functions.sh                | 44 ++++++++++
 tools/pip/setup.py                            | 14 ++-
 10 files changed, 331 insertions(+), 64 deletions(-)
 create mode 100644 cd/python/pypi/Jenkins_pipeline.groovy
 create mode 100644 cd/python/pypi/README.md
 create mode 100755 cd/python/pypi/pypi_package.sh
 create mode 100755 cd/python/pypi/pypi_publish.py
 delete mode 100644 ci/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy

diff --git a/cd/Jenkinsfile_cd_pipeline b/cd/Jenkinsfile_cd_pipeline
index 9aa5f8110d48..e0e94770b682 100644
--- a/cd/Jenkinsfile_cd_pipeline
+++ b/cd/Jenkinsfile_cd_pipeline
@@ -61,6 +61,10 @@ pipeline {
               stage("Build") {
                 cd_utils.trigger_release_job("Build static libmxnet", "mxnet_lib/static", params.MXNET_VARIANTS)    
               }
+              stage("PyPI Release") {
+                echo "Building PyPI Release"
+                cd_utils.trigger_release_job("Release PyPI Packages", "python/pypi", params.MXNET_VARIANTS)
+              }
             },
 
             "Dynamic libmxnet based release": {
diff --git a/cd/Jenkinsfile_utils.groovy b/cd/Jenkinsfile_utils.groovy
index 370e746cb954..5182b04a3b5b 100644
--- a/cd/Jenkinsfile_utils.groovy
+++ b/cd/Jenkinsfile_utils.groovy
@@ -121,14 +121,15 @@ def push_artifact(libmxnet_path, variant, libtype, license_paths = '', dependenc
 
 // pull artifact from repository
 def pull_artifact(variant, libtype, destination = '') {
-  sh "./ci/cd/utils/artifact_repository.py --pull --verbose --libtype ${libtype} --variant ${variant} --destination ${destination}"
+  sh "./cd/utils/artifact_repository.py --pull --verbose --libtype ${libtype} --variant ${variant} --destination ${destination}"
 }
 
 // pulls artifact from repository and places files in the appropriate directories
 def restore_artifact(variant, libtype) {
+
   pull_artifact(variant, libtype, 'mxnet_artifact')
-  // move libraries to lib directory
 
+  // move libraries to lib directory
   dir('lib') {
     sh "mv ../mxnet_artifact/libmxnet.so ."
     if (fileExists('../mxnet_artifact/dependencies')) {
@@ -160,13 +161,13 @@ def restore_artifact(variant, libtype) {
 }
 
 // A generic pipeline that can be used by *most* CD jobs
-// It can use used when implementing the pipeline steps in the Jenkins_steps.groovy
-// script for the particular delivery channel. However, it should also implemente the
+// It can be used when implementing the pipeline steps in the Jenkins_steps.groovy
+// script for a particular delivery channel. However, it should also implement the
 // build, test, and push steps.
 // NOTE: Be mindful of the expected time that a step should take. If it will take a long time,
 // and it can be done in a CPU node, do it in a CPU node. We should avoid using GPU instances unless
 // we *have* to.
-// However, if it is only packaging libmxnet and that doesn't take long. The the pipeline can 
+// However, if it is only packaging libmxnet and that doesn't take long. Then, the pipeline can 
 // just run on a single node. As is done bellow.
 // For examples of multi-node CD pipelines, see the the binary_release/static and binary_release/dynamic
 // pipeline.
diff --git a/cd/README.md b/cd/README.md
index 41188eee6785..a8e3fb794daf 100644
--- a/cd/README.md
+++ b/cd/README.md
@@ -159,6 +159,10 @@ def get_pipeline(mxnet_variant) {
 }
 ```
 
+Examples:
+
+ * [PyPI Release](python/pypi/Jenkins_pipeline.groovy): In this pipeline, the majority of time is overwhelmingly spent on testing. Therefore, it should be ok to execute the whole pipeline on a GPU node (i.e. packaging, testing, and publishing).
+
 **Per step**
 
 Use this approach in cases where you have long running stages that don't depend on specialized/expensive hardware.
@@ -188,3 +192,7 @@ def test(mxnet_variant) {
   }
 }
 ```
+
+Examples:
+
+Both the [statically linked libmxnet](mxnet_lib/static/Jenkins_pipeline.groovy) and [dynamically linked libmxnet](mxnet_lib/dynamic/Jenkins_pipeline.groovy) pipelines have long running compilation and testing stages that **do not** require specialized/expensive hardware (e.g. GPUs). Therefore, as mush as possible, it is important to run each stage in on its own node, and design the pipeline to spend the least amount of time possible on expensive hardware. E.g. for GPU builds, only run GPU tests on GPU instances, all other stages can be executed on CPU nodes.
\ No newline at end of file
diff --git a/cd/python/pypi/Jenkins_pipeline.groovy b/cd/python/pypi/Jenkins_pipeline.groovy
new file mode 100644
index 000000000000..a89228aef80b
--- /dev/null
+++ b/cd/python/pypi/Jenkins_pipeline.groovy
@@ -0,0 +1,68 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// NOTE: 
+// ci_utils and cd_utils are loaded by the originating Jenkins job, e.g. jenkins/Jenkinsfile_release_job
+
+def get_pipeline(mxnet_variant) {
+  def node_type = mxnet_variant.startsWith('cu') ? NODE_LINUX_GPU : NODE_LINUX_CPU
+  return cd_utils.generic_pipeline(mxnet_variant, this, node_type)
+}
+
+def get_environment(mxnet_variant) {
+  def environment = "ubuntu_cpu"
+  if (mxnet_variant.startsWith('cu')) {
+    environment = "ubuntu_gpu_${mxnet_variant}".replace("mkl", "")
+  }
+  return environment
+}
+
+def build(mxnet_variant) {
+  ws("workspace/python_pypi/${mxnet_variant}/${env.BUILD_NUMBER}") {
+    ci_utils.init_git()
+    cd_utils.restore_artifact(mxnet_variant, 'static')
+    
+    // create wheel file
+    def environment = get_environment(mxnet_variant)
+    def nvidia_docker = mxnet_variant.startsWith('cu')
+    ci_utils.docker_run(environment, "cd_package_pypi ${mxnet_variant}", nvidia_docker, '500m', "RELEASE_BUILD='${env.RELEASE_BUILD}'")
+  }
+}
+
+def test(mxnet_variant) {
+  ws("workspace/python_pypi/${mxnet_variant}/${env.BUILD_NUMBER}") {
+    // test wheel file
+    def environment = get_environment(mxnet_variant)
+    def nvidia_docker = mxnet_variant.startsWith('cu')
+    ci_utils.docker_run(environment, "cd_integration_test_pypi python ${nvidia_docker}", nvidia_docker)
+    ci_utils.docker_run(environment, "cd_integration_test_pypi python3 ${nvidia_docker}", nvidia_docker)
+  }
+}
+
+def push(mxnet_variant) {
+  ws("workspace/python_pypi/${mxnet_variant}/${env.BUILD_NUMBER}") {
+    // publish package to pypi
+    sh "./ci/docker/runtime_functions.sh cd_pypi_publish"
+  }
+}
+
+return this
diff --git a/cd/python/pypi/README.md b/cd/python/pypi/README.md
new file mode 100644
index 000000000000..4a17a00923c8
--- /dev/null
+++ b/cd/python/pypi/README.md
@@ -0,0 +1,43 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+# PyPI CD Pipeline
+
+The Jenkins pipelines for continuous delivery of the PyPI MXNet packages.
+The pipelines for each variant are run, and fail, independently. Each depends
+on a successful build of the statically linked libmxet library.
+
+The pipeline relies on the scripts and resources located in [tools/pip](https://github.com/apache/incubator-mxnet/tree/master/tools/pip)
+to build the PyPI packages.
+
+## Credentials
+
+The pipeline depends on the following environment variables in order to successfully
+retrieve the credentials for the PyPI account:
+
+* CD_PYPI_SECRET_NAME
+* DOCKERHUB_SECRET_ENDPOINT_URL
+* DOCKERHUB_SECRET_ENDPOINT_REGION
+
+The credentials are stored in the Secrets Manager of the AWS account hosting Jenkins.
+The [pypi_publish.py](pypi_publish.sh) script is in charge of retrieving the credentials.
+
+## Mock publishing
+
+Because of space limitations on PyPI, we don't want to push test packages from Jenkins Dev
+everytime the pipeline is run. Therefore, the [pypi_publish.sh](pypi_publish.sh) 
+script will fake publishing packages if the `username` is *skipPublish*.
diff --git a/cd/python/pypi/pypi_package.sh b/cd/python/pypi/pypi_package.sh
new file mode 100755
index 000000000000..fdfed461eb44
--- /dev/null
+++ b/cd/python/pypi/pypi_package.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+
+# variant = cpu, mkl, cu80, cu80mkl, cu100, etc.
+export mxnet_variant=${1:?"Please specify the mxnet variant"}
+
+# Due to this PR: https://github.com/apache/incubator-mxnet/pull/14899
+# The setup.py expects that mkldnn_version.h be present in 
+# mxnet-build/3rdparty/mkldnn/build/install/include
+# The artifact repository stores this file in the dependencies
+# and CD unpacks it to a directory called cd_misc
+if [ -f "cd_misc/mkldnn_version.h" ]; then
+  mkdir -p 3rdparty/mkldnn/build/install/include
+  cp cd_misc/mkldnn_version.h 3rdparty/mkldnn/build/install/include/.
+fi
+
+# Create wheel workspace
+rm -rf wheel_build
+mkdir wheel_build
+cd wheel_build
+
+# Setup workspace
+# setup.py expects mxnet-build to be the
+# mxnet directory
+ln -s ../. mxnet-build
+
+# Copy the setup.py and other package resources
+cp -R ../tools/pip/* .
+
+# Remove comment lines from pip doc files
+pushd doc
+for file in $(ls); do
+  sed -i '/<!--/d' ${file}
+done
+popd
+
+echo "Building python package with environment:"
+printenv
+echo "-----------------------------------------"
+
+# Build wheel file - placed in wheel_build/dist
+python setup.py bdist_wheel
diff --git a/cd/python/pypi/pypi_publish.py b/cd/python/pypi/pypi_publish.py
new file mode 100755
index 000000000000..94d6e8703a52
--- /dev/null
+++ b/cd/python/pypi/pypi_publish.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import logging
+import os
+import subprocess
+import sys
+
+import boto3
+from botocore.exceptions import ClientError
+
+
+def post_wheel(path):
+    """
+    Posts mxnet wheel file to PyPI
+    """
+    logging.info('Posting {} to PyPI'.format(path))
+    pypi_credentials = get_secret()
+
+    cmd = 'python3 -m twine upload --username {} --password {} {}'.format(
+        pypi_credentials['username'],
+        pypi_credentials['password'],
+        path)
+
+    # The PyPI credentials for DEV has username set to 'skipPublish'
+    # This way we do not attempt to publish the PyPI package
+    # Just print a helpful message
+    if pypi_credentials['username'] == 'skipPublish':
+        print('In DEV account, skipping publish')
+        print('Would have run: {}'.format(cmd))
+        return 0
+    else:
+        # DO NOT PRINT CMD IN THIS BLOCK, includes password
+        p = subprocess.run(cmd.split(' '),
+                        stdout=subprocess.PIPE)
+        logging.info(p.stdout)
+        return p.returncode
+
+def get_secret():
+    secret_name = os.environ['CD_PYPI_SECRET_NAME']
+    endpoint_url = os.environ['DOCKERHUB_SECRET_ENDPOINT_URL']
+    region_name = os.environ['DOCKERHUB_SECRET_ENDPOINT_REGION']
+
+    session = boto3.Session()
+    client = session.client(
+        service_name='secretsmanager',
+        region_name=region_name,
+        endpoint_url=endpoint_url
+    )
+
+    try:
+        get_secret_value_response = client.get_secret_value(SecretId=secret_name)
+    except ClientError as e:
+        if e.response['Error']['Code'] == 'DecryptionFailureException':
+            raise e
+        elif e.response['Error']['Code'] == 'InternalServiceErrorException':
+            raise e
+        elif e.response['Error']['Code'] == 'InvalidParameterException':
+            raise e
+        elif e.response['Error']['Code'] == 'InvalidRequestException':
+            raise e
+        elif e.response['Error']['Code'] == 'ResourceNotFoundException':
+            raise e
+    else:
+        return json.loads(get_secret_value_response['SecretString'])
+        
+            
+if __name__ == '__main__':
+    sys.exit(post_wheel(sys.argv[1]))
diff --git a/ci/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy b/ci/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy
deleted file mode 100644
index 0e70e4d9dbe5..000000000000
--- a/ci/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy
+++ /dev/null
@@ -1,57 +0,0 @@
-// -*- mode: groovy -*-
-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-// Jenkins pipeline
-// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
-// NOTE: ci_utils is loaded by the originating Jenkins job, e.g. jenkins/Jenkinsfile_release_job
-
-// libmxnet location
-libmxnet = 'lib/libmxnet.so'
-
-// licenses
-licenses = 'licenses/*'
-
-// libmxnet dependencies
-mx_deps = ''
-mx_mkldnn_deps = 'lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so'
-
-// library type
-// either static or dynamic - depending on how it links to its dependencies
-libtype = 'dynamic'
-
-libmxnet_pipeline = load('ci/cd/mxnet_lib/mxnet_lib_pipeline.groovy')
-
-// Builds the dynamic binary for the specified mxnet variant
-def build(mxnet_variant) {
-  node(NODE_LINUX_CPU) {
-    ws("workspace/mxnet_${libtype}/${mxnet_variant}/${env.BUILD_NUMBER}") {
-      def image = libmxnet_pipeline.get_environment(mxnet_variant)
-      ci_utils.init_git()
-      ci_utils.docker_run(image, "build_dynamic_libmxnet ${mxnet_variant}", false)
-      ci_utils.pack_lib("mxnet_${mxnet_variant}", libmxnet_pipeline.get_stash(mxnet_variant))
-    }
-  }
-}
-
-def get_pipeline(mxnet_variant) {
-  return libmxnet_pipeline.get_pipeline(mxnet_variant, this.&build)
-}
-
-return this
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 7d91d4e5d121..83b7304edb2c 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -1886,6 +1886,50 @@ build_static_libmxnet() {
     popd
 }
 
+# Packages libmxnet into wheel file
+cd_package_pypi() {
+    set -ex
+    pushd .
+    local mxnet_variant=${1:?"This function requires a python command as the first argument"}
+    ./cd/python/pypi/pypi_package.sh ${mxnet_variant}
+    popd
+}
+
+# Sanity checks wheel file 
+cd_integration_test_pypi() {
+    set -ex
+    local python_cmd=${1:?"This function requires a python command as the first argument"}
+    local gpu_enabled=${2:-"false"}
+
+    local test_conv_params=''
+    local mnist_params=''
+
+    local pip_cmd='pip'
+
+    if [ "${gpu_enabled}" = "true" ]; then
+        mnist_params="--gpu 0"
+        test_conv_params="--gpu"
+    fi
+
+    if [ "${python_cmd}" = "python3" ]; then
+        pip_cmd='pip3'
+    fi
+
+    # install mxnet wheel package
+    ${pip_cmd} install --user ./wheel_build/dist/*.whl
+
+    # execute tests
+    ${python_cmd} /work/mxnet/tests/python/train/test_conv.py ${test_conv_params}
+    ${python_cmd} /work/mxnet/example/image-classification/train_mnist.py ${mnist_params}
+}
+
+# Publishes wheel to PyPI
+cd_pypi_publish() {
+    set -ex
+    pip3 install --user twine
+    ./cd/python/pypi/pypi_publish.py `readlink -f wheel_build/dist/*.whl`
+}
+
 build_static_scala_mkl() {
     set -ex
     pushd .
diff --git a/tools/pip/setup.py b/tools/pip/setup.py
index f0a3e37fd3c3..82aa632d28ad 100644
--- a/tools/pip/setup.py
+++ b/tools/pip/setup.py
@@ -41,9 +41,19 @@
 
 LIB_PATH = libinfo['find_lib_path']()
 __version__ = libinfo['__version__']
-if 'TRAVIS_TAG' not in os.environ or not os.environ['TRAVIS_TAG'].strip():
+
+# set by the CD pipeline
+is_release = os.environ.get("IS_RELEASE", "").strip()
+
+# set by the travis build pipeline
+travis_tag = os.environ.get("TRAVIS_TAG", "").strip()
+
+# nightly build tag
+if not travis_tag and not is_release:
     __version__ += 'b{0}'.format(datetime.today().strftime('%Y%m%d'))
-elif 'TRAVIS_TAG' in os.environ and os.environ['TRAVIS_TAG'].startswith('patch-'):
+
+# patch build tag
+elif travis_tag.startswith('patch-'):
     __version__ = os.environ['TRAVIS_TAG'].split('-')[1]
 
 class BinaryDistribution(Distribution):

From ad881f9ae79133014c6cc157df3ad74528ab574a Mon Sep 17 00:00:00 2001
From: Per Goncalves da Silva <perdasilva@gmail.com>
Date: Wed, 25 Sep 2019 07:21:13 +0200
Subject: [PATCH 3/5] Adds pypi packaging fixes

---
 cd/python/pypi/pypi_package.sh       |  1 +
 tools/pip/doc/CU101MKL_ADDITIONAL.md | 46 ++++++++++++++++++++++++++++
 tools/pip/doc/CU101_ADDITIONAL.md    | 46 ++++++++++++++++++++++++++++
 3 files changed, 93 insertions(+)
 create mode 100644 tools/pip/doc/CU101MKL_ADDITIONAL.md
 create mode 100644 tools/pip/doc/CU101_ADDITIONAL.md

diff --git a/cd/python/pypi/pypi_package.sh b/cd/python/pypi/pypi_package.sh
index fdfed461eb44..581bf694cb31 100755
--- a/cd/python/pypi/pypi_package.sh
+++ b/cd/python/pypi/pypi_package.sh
@@ -54,6 +54,7 @@ popd
 echo "Building python package with environment:"
 printenv
 echo "-----------------------------------------"
+pip install --user pypandoc
 
 # Build wheel file - placed in wheel_build/dist
 python setup.py bdist_wheel
diff --git a/tools/pip/doc/CU101MKL_ADDITIONAL.md b/tools/pip/doc/CU101MKL_ADDITIONAL.md
new file mode 100644
index 000000000000..7304084f6bad
--- /dev/null
+++ b/tools/pip/doc/CU101MKL_ADDITIONAL.md
@@ -0,0 +1,46 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+Prerequisites
+-------------
+This package supports Linux and Windows platforms. You may also want to check:
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101/) with CUDA-10.1 support.
+- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100/) with CUDA-10.0 support.
+- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
+- [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
+- [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
+- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
+- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
+- [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
+- [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
+- [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
+- [mxnet-cu80mkl](https://pypi.python.org/pypi/mxnet-cu80mkl/) with CUDA-8.0 support and MKLDNN support.
+- [mxnet-cu75](https://pypi.python.org/pypi/mxnet-cu75/) with CUDA-7.5 support.
+- [mxnet-cu75mkl](https://pypi.python.org/pypi/mxnet-cu75mkl/) with CUDA-7.5 support and MKLDNN support.
+- [mxnet-mkl](https://pypi.python.org/pypi/mxnet-mkl/) with MKLDNN support.
+- [mxnet](https://pypi.python.org/pypi/mxnet/).
+
+To download CUDA, check [CUDA download](https://developer.nvidia.com/cuda-downloads). For more instructions, check [CUDA Toolkit online documentation](http://docs.nvidia.com/cuda/index.html).
+
+To install for other platforms (e.g. Windows, Raspberry Pi/ARM) or other versions of CUDA, check [Installing MXNet](https://mxnet.incubator.apache.org/versions/master/install/index.html) for instructions on building from source.
+
+Installation
+------------
+To install:
+```bash
+pip install mxnet-cu100mkl
+```
diff --git a/tools/pip/doc/CU101_ADDITIONAL.md b/tools/pip/doc/CU101_ADDITIONAL.md
new file mode 100644
index 000000000000..4f66c2650653
--- /dev/null
+++ b/tools/pip/doc/CU101_ADDITIONAL.md
@@ -0,0 +1,46 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+Prerequisites
+-------------
+This package supports Linux and Windows platforms. You may also want to check:
+- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
+- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
+- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
+- [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
+- [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
+- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
+- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
+- [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
+- [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
+- [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
+- [mxnet-cu80mkl](https://pypi.python.org/pypi/mxnet-cu80mkl/) with CUDA-8.0 support and MKLDNN support.
+- [mxnet-cu75](https://pypi.python.org/pypi/mxnet-cu75/) with CUDA-7.5 support.
+- [mxnet-cu75mkl](https://pypi.python.org/pypi/mxnet-cu75mkl/) with CUDA-7.5 support and MKLDNN support.
+- [mxnet-mkl](https://pypi.python.org/pypi/mxnet-mkl/) with MKLDNN support.
+- [mxnet](https://pypi.python.org/pypi/mxnet/).
+
+To download CUDA, check [CUDA download](https://developer.nvidia.com/cuda-downloads). For more instructions, check [CUDA Toolkit online documentation](http://docs.nvidia.com/cuda/index.html).
+
+To install for other platforms (e.g. Windows, Raspberry Pi/ARM) or other versions, check [Installing MXNet](https://mxnet.incubator.apache.org/versions/master/install/index.html) for instructions on building from source.
+
+Installation
+------------
+To install:
+```bash
+pip install mxnet-cu100
+```

From e77525b7297a9ba16bd08df6d3872bda315e6953 Mon Sep 17 00:00:00 2001
From: Per Goncalves da Silva <perdasilva@gmail.com>
Date: Mon, 30 Sep 2019 09:19:59 +0200
Subject: [PATCH 4/5] Only publish cu92 and cu92mkl variants to PyPI

---
 cd/python/pypi/Jenkins_pipeline.groovy | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/cd/python/pypi/Jenkins_pipeline.groovy b/cd/python/pypi/Jenkins_pipeline.groovy
index a89228aef80b..bf8103270146 100644
--- a/cd/python/pypi/Jenkins_pipeline.groovy
+++ b/cd/python/pypi/Jenkins_pipeline.groovy
@@ -23,6 +23,12 @@
 // NOTE: 
 // ci_utils and cd_utils are loaded by the originating Jenkins job, e.g. jenkins/Jenkinsfile_release_job
 
+// Only post the following variants to PyPI.
+// This is a temporary solution until we are confident with the packages generated by CI
+// This should be removed in the not too distant future.
+// We only skip the publish step so we can still QA the other variants.
+pypi_releases = ["cu92", "cu92mkl"]
+
 def get_pipeline(mxnet_variant) {
   def node_type = mxnet_variant.startsWith('cu') ? NODE_LINUX_GPU : NODE_LINUX_CPU
   return cd_utils.generic_pipeline(mxnet_variant, this, node_type)
@@ -61,7 +67,11 @@ def test(mxnet_variant) {
 def push(mxnet_variant) {
   ws("workspace/python_pypi/${mxnet_variant}/${env.BUILD_NUMBER}") {
     // publish package to pypi
-    sh "./ci/docker/runtime_functions.sh cd_pypi_publish"
+    if (mxnet_variant in pypi_releases) {
+      sh "./ci/docker/runtime_functions.sh cd_pypi_publish"
+    } else {
+      echo "Temporarily skipping publishing PyPI package for '${mxnet_variant}'."
+    }
   }
 }
 

From 50952bf241db1c8a141f8b77203f222aec887ec0 Mon Sep 17 00:00:00 2001
From: Per Goncalves da Silva <perdasilva@gmail.com>
Date: Mon, 30 Sep 2019 18:36:36 +0200
Subject: [PATCH 5/5] Updates pip package doc resources

---
 tools/pip/doc/CPU_ADDITIONAL.md      |  6 ++--
 tools/pip/doc/CU100MKL_ADDITIONAL.md |  4 +--
 tools/pip/doc/CU100_ADDITIONAL.md    |  4 +--
 tools/pip/doc/CU101MKL_ADDITIONAL.md |  2 --
 tools/pip/doc/CU101_ADDITIONAL.md    |  2 --
 tools/pip/doc/CU75MKL_ADDITIONAL.md  |  6 ++--
 tools/pip/doc/CU75_ADDITIONAL.md     |  6 ++--
 tools/pip/doc/CU80MKL_ADDITIONAL.md  |  6 ++--
 tools/pip/doc/CU80_ADDITIONAL.md     |  6 ++--
 tools/pip/doc/CU90MKL_ADDITIONAL.md  |  6 ++--
 tools/pip/doc/CU90_ADDITIONAL.md     |  6 ++--
 tools/pip/doc/CU91MKL_ADDITIONAL.md  | 42 ----------------------------
 tools/pip/doc/CU91_ADDITIONAL.md     | 42 ----------------------------
 tools/pip/doc/CU92MKL_ADDITIONAL.md  |  6 ++--
 tools/pip/doc/CU92_ADDITIONAL.md     |  6 ++--
 tools/pip/doc/MKL_ADDITIONAL.md      |  6 ++--
 16 files changed, 44 insertions(+), 112 deletions(-)
 delete mode 100644 tools/pip/doc/CU91MKL_ADDITIONAL.md
 delete mode 100644 tools/pip/doc/CU91_ADDITIONAL.md

diff --git a/tools/pip/doc/CPU_ADDITIONAL.md b/tools/pip/doc/CPU_ADDITIONAL.md
index 9f25121e5b8c..bbd747dbf557 100644
--- a/tools/pip/doc/CPU_ADDITIONAL.md
+++ b/tools/pip/doc/CPU_ADDITIONAL.md
@@ -18,10 +18,12 @@
 Prerequisites
 -------------
 This package supports Linux, Mac OSX, and Windows platforms. You may also want to check:
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
+- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
+- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
+- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
 - [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
 - [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
diff --git a/tools/pip/doc/CU100MKL_ADDITIONAL.md b/tools/pip/doc/CU100MKL_ADDITIONAL.md
index a916f0b9583b..80690ea8b964 100644
--- a/tools/pip/doc/CU100MKL_ADDITIONAL.md
+++ b/tools/pip/doc/CU100MKL_ADDITIONAL.md
@@ -18,11 +18,11 @@
 Prerequisites
 -------------
 This package supports Linux and Windows platforms. You may also want to check:
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
+- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
 - [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100/) with CUDA-10.0 support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
 - [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
 - [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
diff --git a/tools/pip/doc/CU100_ADDITIONAL.md b/tools/pip/doc/CU100_ADDITIONAL.md
index 1a1f4dd70ea2..de5e62051cc3 100644
--- a/tools/pip/doc/CU100_ADDITIONAL.md
+++ b/tools/pip/doc/CU100_ADDITIONAL.md
@@ -18,11 +18,11 @@
 Prerequisites
 -------------
 This package supports Linux and Windows platforms. You may also want to check:
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
+- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
 - [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
 - [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
 - [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
diff --git a/tools/pip/doc/CU101MKL_ADDITIONAL.md b/tools/pip/doc/CU101MKL_ADDITIONAL.md
index 7304084f6bad..9cdebe3dddbb 100644
--- a/tools/pip/doc/CU101MKL_ADDITIONAL.md
+++ b/tools/pip/doc/CU101MKL_ADDITIONAL.md
@@ -23,8 +23,6 @@ This package supports Linux and Windows platforms. You may also want to check:
 - [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
 - [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
 - [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
diff --git a/tools/pip/doc/CU101_ADDITIONAL.md b/tools/pip/doc/CU101_ADDITIONAL.md
index 4f66c2650653..f58413c09e72 100644
--- a/tools/pip/doc/CU101_ADDITIONAL.md
+++ b/tools/pip/doc/CU101_ADDITIONAL.md
@@ -23,8 +23,6 @@ This package supports Linux and Windows platforms. You may also want to check:
 - [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
 - [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
 - [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
diff --git a/tools/pip/doc/CU75MKL_ADDITIONAL.md b/tools/pip/doc/CU75MKL_ADDITIONAL.md
index 30b85fda0c2e..dc485d687f1e 100644
--- a/tools/pip/doc/CU75MKL_ADDITIONAL.md
+++ b/tools/pip/doc/CU75MKL_ADDITIONAL.md
@@ -18,10 +18,12 @@
 Prerequisites
 -------------
 This package supports Linux only, up to 1.2.1. You may also want to check:
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
+- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
+- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
+- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
 - [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
 - [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
diff --git a/tools/pip/doc/CU75_ADDITIONAL.md b/tools/pip/doc/CU75_ADDITIONAL.md
index 9947365517e6..a5257913adc5 100644
--- a/tools/pip/doc/CU75_ADDITIONAL.md
+++ b/tools/pip/doc/CU75_ADDITIONAL.md
@@ -18,10 +18,12 @@
 Prerequisites
 -------------
 This package supports Linux only, up to 1.2.1. You may also want to check:
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
+- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
+- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
+- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
 - [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
 - [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
diff --git a/tools/pip/doc/CU80MKL_ADDITIONAL.md b/tools/pip/doc/CU80MKL_ADDITIONAL.md
index 54c54536ee2b..24cec2b93668 100644
--- a/tools/pip/doc/CU80MKL_ADDITIONAL.md
+++ b/tools/pip/doc/CU80MKL_ADDITIONAL.md
@@ -18,10 +18,12 @@
 Prerequisites
 -------------
 This package supports Linux and Windows platforms. You may also want to check:
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
+- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
+- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
+- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
 - [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
 - [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
diff --git a/tools/pip/doc/CU80_ADDITIONAL.md b/tools/pip/doc/CU80_ADDITIONAL.md
index 609567c72902..767f034e120e 100644
--- a/tools/pip/doc/CU80_ADDITIONAL.md
+++ b/tools/pip/doc/CU80_ADDITIONAL.md
@@ -18,10 +18,12 @@
 Prerequisites
 -------------
 This package supports Linux and Windows platforms. You may also want to check:
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
+- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
+- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
+- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
 - [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
 - [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80mkl](https://pypi.python.org/pypi/mxnet-cu80mkl/) with CUDA-8.0 support and MKLDNN support.
diff --git a/tools/pip/doc/CU90MKL_ADDITIONAL.md b/tools/pip/doc/CU90MKL_ADDITIONAL.md
index 7fe04dae29ad..66f18d35ef73 100644
--- a/tools/pip/doc/CU90MKL_ADDITIONAL.md
+++ b/tools/pip/doc/CU90MKL_ADDITIONAL.md
@@ -18,10 +18,12 @@
 Prerequisites
 -------------
 This package supports Linux and Windows platforms. You may want to check:
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
+- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
+- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
+- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
 - [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
 - [mxnet-cu80mkl](https://pypi.python.org/pypi/mxnet-cu80mkl/) with CUDA-8.0 support and MKLDNN support.
diff --git a/tools/pip/doc/CU90_ADDITIONAL.md b/tools/pip/doc/CU90_ADDITIONAL.md
index 37ab45a7dfbf..f21b77019d12 100644
--- a/tools/pip/doc/CU90_ADDITIONAL.md
+++ b/tools/pip/doc/CU90_ADDITIONAL.md
@@ -18,10 +18,12 @@
 Prerequisites
 -------------
 This package supports Linux and Windows platforms. You may also want to check:
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
+- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
+- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
+- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
 - [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
 - [mxnet-cu80mkl](https://pypi.python.org/pypi/mxnet-cu80mkl/) with CUDA-8.0 support and MKLDNN support.
diff --git a/tools/pip/doc/CU91MKL_ADDITIONAL.md b/tools/pip/doc/CU91MKL_ADDITIONAL.md
deleted file mode 100644
index 8e82b5a18868..000000000000
--- a/tools/pip/doc/CU91MKL_ADDITIONAL.md
+++ /dev/null
@@ -1,42 +0,0 @@
-<!--- Licensed to the Apache Software Foundation (ASF) under one -->
-<!--- or more contributor license agreements.  See the NOTICE file -->
-<!--- distributed with this work for additional information -->
-<!--- regarding copyright ownership.  The ASF licenses this file -->
-<!--- to you under the Apache License, Version 2.0 (the -->
-<!--- "License"); you may not use this file except in compliance -->
-<!--- with the License.  You may obtain a copy of the License at -->
-
-<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
-
-<!--- Unless required by applicable law or agreed to in writing, -->
-<!--- software distributed under the License is distributed on an -->
-<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
-<!--- KIND, either express or implied.  See the License for the -->
-<!--- specific language governing permissions and limitations -->
-<!--- under the License. -->
-
-Prerequisites
--------------
-This package supports Linux and Windows platforms, up to 1.2.1. You may also want to check:
-- [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
-- [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
-- [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
-- [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
-- [mxnet-cu80mkl](https://pypi.python.org/pypi/mxnet-cu80mkl/) with CUDA-8.0 support and MKLDNN support.
-- [mxnet-cu75](https://pypi.python.org/pypi/mxnet-cu75/) with CUDA-7.5 support.
-- [mxnet-cu75mkl](https://pypi.python.org/pypi/mxnet-cu75mkl/) with CUDA-7.5 support and MKLDNN support.
-- [mxnet-mkl](https://pypi.python.org/pypi/mxnet-mkl/) with MKLDNN support.
-- [mxnet](https://pypi.python.org/pypi/mxnet/).
-
-To download CUDA, check [CUDA download](https://developer.nvidia.com/cuda-downloads). For more instructions, check [CUDA Toolkit online documentation](http://docs.nvidia.com/cuda/index.html).
-
-To install for other platforms (e.g. Windows, Raspberry Pi/ARM) or other versions of CUDA, check [Installing MXNet](https://mxnet.apache.org/versions/master/install/index.html) for instructions on building from source.
-
-Installation
-------------
-To install:
-```bash
-pip install mxnet-cu91mkl
-```
diff --git a/tools/pip/doc/CU91_ADDITIONAL.md b/tools/pip/doc/CU91_ADDITIONAL.md
deleted file mode 100644
index 5918a8901f99..000000000000
--- a/tools/pip/doc/CU91_ADDITIONAL.md
+++ /dev/null
@@ -1,42 +0,0 @@
-<!--- Licensed to the Apache Software Foundation (ASF) under one -->
-<!--- or more contributor license agreements.  See the NOTICE file -->
-<!--- distributed with this work for additional information -->
-<!--- regarding copyright ownership.  The ASF licenses this file -->
-<!--- to you under the Apache License, Version 2.0 (the -->
-<!--- "License"); you may not use this file except in compliance -->
-<!--- with the License.  You may obtain a copy of the License at -->
-
-<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
-
-<!--- Unless required by applicable law or agreed to in writing, -->
-<!--- software distributed under the License is distributed on an -->
-<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
-<!--- KIND, either express or implied.  See the License for the -->
-<!--- specific language governing permissions and limitations -->
-<!--- under the License. -->
-
-Prerequisites
--------------
-This package supports Linux and Windows platforms, up to 1.2.1. You may also want to check:
-- [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
-- [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
-- [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
-- [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
-- [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
-- [mxnet-cu80mkl](https://pypi.python.org/pypi/mxnet-cu80mkl/) with CUDA-8.0 support and MKLDNN support.
-- [mxnet-cu75](https://pypi.python.org/pypi/mxnet-cu75/) with CUDA-7.5 support.
-- [mxnet-cu75mkl](https://pypi.python.org/pypi/mxnet-cu75mkl/) with CUDA-7.5 support and MKLDNN support.
-- [mxnet-mkl](https://pypi.python.org/pypi/mxnet-mkl/) with MKLDNN support.
-- [mxnet](https://pypi.python.org/pypi/mxnet/).
-
-To download CUDA, check [CUDA download](https://developer.nvidia.com/cuda-downloads). For more instructions, check [CUDA Toolkit online documentation](http://docs.nvidia.com/cuda/index.html).
-
-To install for other platforms (e.g. Windows, Raspberry Pi/ARM) or other versions, check [Installing MXNet](https://mxnet.apache.org/versions/master/install/index.html) for instructions on building from source.
-
-Installation
-------------
-To install:
-```bash
-pip install mxnet-cu91
-```
diff --git a/tools/pip/doc/CU92MKL_ADDITIONAL.md b/tools/pip/doc/CU92MKL_ADDITIONAL.md
index f7adbfcb61ca..a4e64f55bde9 100644
--- a/tools/pip/doc/CU92MKL_ADDITIONAL.md
+++ b/tools/pip/doc/CU92MKL_ADDITIONAL.md
@@ -18,9 +18,11 @@
 Prerequisites
 -------------
 This package supports Linux and Windows platforms. You may also want to check:
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
+- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
+- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
+- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
 - [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
diff --git a/tools/pip/doc/CU92_ADDITIONAL.md b/tools/pip/doc/CU92_ADDITIONAL.md
index 79afccaec095..849245de72c9 100644
--- a/tools/pip/doc/CU92_ADDITIONAL.md
+++ b/tools/pip/doc/CU92_ADDITIONAL.md
@@ -18,9 +18,11 @@
 Prerequisites
 -------------
 This package supports Linux and Windows platforms. You may also want to check:
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
+- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
+- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
+- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
 - [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
 - [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
diff --git a/tools/pip/doc/MKL_ADDITIONAL.md b/tools/pip/doc/MKL_ADDITIONAL.md
index d796aca1b82c..d6236cf71ce7 100644
--- a/tools/pip/doc/MKL_ADDITIONAL.md
+++ b/tools/pip/doc/MKL_ADDITIONAL.md
@@ -18,10 +18,12 @@
 Prerequisites
 -------------
 This package supports Linux only. You may also want to check:
+- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
+- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
+- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
+- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
 - [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
 - [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
-- [mxnet-cu91](https://pypi.python.org/pypi/mxnet-cu91/) with CUDA-9.1 support.
-- [mxnet-cu91mkl](https://pypi.python.org/pypi/mxnet-cu91mkl/) with CUDA-9.1 support and MKLDNN support.
 - [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
 - [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
 - [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.