diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index a0795eb58a5a..8805850e3145 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -814,7 +814,11 @@ build_docs() {
     set -ex
     pushd .
     cd /work/mxnet/docs/build_version_doc
-    ./build_all_version.sh $1
+    # Parameters are set in the Jenkins pipeline: restricted-website-build
+    # $1 is the list of branches to build; $2 is the list of tags to display
+    # So you can build from the 1.2.0 branch, but display 1.2.1 on the site
+    ./build_all_version.sh $1 $2
+    # $3 is the default version tag for the website; $4 is the base URL
     ./update_all_version.sh $2 $3 $4
     cd VersionedWeb
     tar -zcvf ../artifacts.tgz .
diff --git a/docs/build_version_doc/build_all_version.sh b/docs/build_version_doc/build_all_version.sh
index 56b80e3a0fc8..44cd540fda0b 100755
--- a/docs/build_version_doc/build_all_version.sh
+++ b/docs/build_version_doc/build_all_version.sh
@@ -20,23 +20,42 @@
 # This script is for locally building website for all versions
 # Built files are stored in $built
 
-# Takes one argument:
-# * tag list - space delimited list of Github tags; Example: "1.1.0 1.0.0 master"
+# Takes two arguments:
+# tag list - semicolon delimited list of Github tags
+#   Example: "1.2.0;1.1.0;master"
+# display list - semicolon delimited list of what to display on website
+#   Example: "1.2.1;1.1.0;master"
+# The number of tags for the two arguments must be the same.
 # Example Usage:
-# ./build_all_version.sh "1.1.0 1.0.0 master"
+#   ./build_all_version.sh "1.2.0;1.1.0;master" "1.2.1;1.1.0;master"
+#   ./build_all_version.sh "1.2.0" "1.2.1"
 
 set -e
 set -x
 
 if [ -z "$1" ]
   then
-    echo "Please provide a list of version tags you wish to run."
+    echo "Please provide a list of branches or tags you wish to build."
     exit 1
   else
     IFS=$';'
     tag_list=$1
     echo "Using these tags: $tag_list"
-    for tag in $tag_list; do echo $tag; done
+    build_arr=($tag_list)
+fi
+
+if [ -z "$2" ]
+  then
+    echo "Please provide a list of version tags you wish to display on the site."
+    exit 1
+  else
+    IFS=$';'
+    tags_to_display=$2
+    echo "Displaying these tags: $tags_to_display"
+    display_arr=($tags_to_display)
+    for key in ${!build_arr[@]}; do
+        echo "Branch/tag ${build_arr[${key}]} will be displayed as ${display_arr[${key}]}"
+    done
 fi
 
 mxnet_url="https://github.com/apache/incubator-mxnet.git"
@@ -51,18 +70,27 @@ fi
 if [ ! -d "$built" ]; then
   mkdir $built
   mkdir "$built/versions"
+  else
+    if [ ! -d "$built/versions" ]; then
+      mkdir "$built/versions"
+    fi
 fi
 
-# Build all versions and use latest version(First version number in $tag_list) as landing page.
-for tag in $tag_list; do
+# Checkout each tag and build it
+# Then store it in a folder according to the desired display tag
+for key in ${!build_arr[@]}; do
+    tag=${build_arr[${key}]}
     cd "$mxnet_folder"
     git fetch
     if [ $tag == 'master' ]
         then
             git checkout master
             git pull
+            echo "Building master..."
         else
-            git checkout "v$tag"
+            # Use "v$tag" for branches or pass that in from jenkins
+            git checkout "$tag"
+            echo "Building $tag..."
     fi
 
     git submodule update --init --recursive || exit 1
@@ -72,11 +100,13 @@ for tag in $tag_list; do
     make clean
     make html USE_OPENMP=1 || exit 1
     cd ../../
-    file_loc="$built/versions/$tag"
+    # Use the display tag name for the folder name
+    file_loc="$built/versions/${display_arr[${key}]}"
     if [ -d "$file_loc" ] ; then
         rm -rf "$file_loc"
     fi
     mkdir "$file_loc"
+    echo "Storing artifacts for $tag in $file_loc folder..."
     cp -a "$mxnet_folder/docs/_build/html/." "$file_loc"
 done
 
diff --git a/docs/build_version_doc/update_all_version.sh b/docs/build_version_doc/update_all_version.sh
index bfd656f5ae81..e39b0a503412 100755
--- a/docs/build_version_doc/update_all_version.sh
+++ b/docs/build_version_doc/update_all_version.sh
@@ -23,12 +23,12 @@
 # the tags you want to update.
 
 # Takes three arguments:
-# * tag list - space delimited list of Github tags; Example: "1.1.0 1.0.0 master"
+# * tag list - semicolon delimited list of tags to display on site; Example: "1.1.0;1.0.0;master"
 # * default tag - which version should the site default to; Example: 1.0.0
 # * root URL - for the versions dropdown to change to production or dev server; Example: http://mxnet.incubator.apache.org/
 
 # Example Usage:
-# ./update_all_version.sh "1.1.0 1.0.0 master" 1.0.0 http://mxnet.incubator.apache.org/
+# ./update_all_version.sh "1.1.0;1.0.0;master" 1.0.0 http://mxnet.incubator.apache.org/
 
 set -e
 set -x
@@ -36,7 +36,6 @@ set -x
 MASTER_SOURCE_DIR="../../docs"
 STATIC_FILES_DIR="_static"
 MXNET_THEME_DIR="_static/mxnet-theme"
-BUILD_HTML_DIR="_build/html"
 
 if [ -z "$1" ]
   then
@@ -132,4 +131,3 @@ for tag in $tag_list; do
 done
 
 echo "The output of this process can be found in the VersionedWeb folder."
-