Fixes for CI downloads (apache#14504)

* Fixes for data links * Changed data.dmlc.ml -> data.mxnet.io * Added retries for downloads * Removed silent mode for explicit failure reporting * Reworked download scripts
perdasilva · Mar 26, 2019 · a9458ca · a9458ca
1 parent 056fce4
commit a9458ca
Show file tree

Hide file tree

Showing 22 changed files with 126 additions and 45 deletions.
diff --git a/R-package/tests/testthat/get_data.R b/R-package/tests/testthat/get_data.R
@@ -55,13 +55,16 @@ GetInception <- function() {
   if (!dir.exists("model")) {
     dir.create("model/")
   }
+
   if (!file.exists("model/Inception-BN-0126.params")) {
-    download.file("http://data.dmlc.ml/models/imagenet/inception-bn/Inception-BN-0126.params", 
-      destfile = "model/Inception-BN-0126.params")
+    download.file(
+        "http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-0126.params?raw=true",
+        destfile = "model/Inception-BN-0126.params")
   }
   if (!file.exists("model/Inception-BN-symbol.json")) {
-    download.file("http://data.dmlc.ml/models/imagenet/inception-bn/Inception-BN-symbol.json", 
-      destfile = "model/Inception-BN-symbol.json")
+    download.file(
+        "http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-symbol.json",
+        destfile = "model/Inception-BN-symbol.json")
   }
 }
 

diff --git a/R-package/vignettes/CatsDogsFinetune.Rmd b/R-package/vignettes/CatsDogsFinetune.Rmd
@@ -162,13 +162,13 @@ val   <- data$val
 
 ## Load pretrained model
 
-Here we use the pretrained model from http://data.dmlc.ml/models/imagenet/.
+Here we use the pretrained model from http://data.mxnet.io/mxnet/data/.
 There are 1000 classes in imagenet,
 and we need to replace the last fully connected layer with a new layer for 2 classes.
 
 
 ```{r}
-download.file('http://data.dmlc.ml/data/Inception.zip', destfile = 'Inception.zip')
+download.file('http://data.mxnet.io/mxnet/data/Inception.zip', destfile = 'Inception.zip')
 unzip("Inception.zip")
 inception_bn <- mx.model.load("./Inception-BN", iteration = 126)
 

diff --git a/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd b/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd
@@ -31,7 +31,7 @@ Make sure you unzip the pre-trained model in current folder. And we can use the
 loading function to load the model into R.
 
 ```{r}
-download.file('http://data.dmlc.ml/data/Inception.zip', destfile = 'Inception.zip')
+download.file('http://data.mxnet.io/mxnet/data/Inception.zip', destfile = 'Inception.zip')
 unzip("Inception.zip")
 model <- mx.model.load("Inception/Inception_BN", iteration = 39)
 ```

diff --git a/cpp-package/example/feature_extract/run.sh b/cpp-package/example/feature_extract/run.sh
@@ -17,7 +17,12 @@
 
 # Downloading the data and model
 mkdir -p model
-wget -nc http://data.dmlc.ml/mxnet/models/imagenet/inception-bn.tar.gz
+wget -nc -O model/Inception-BN-symbol.json \
+    http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-symbol.json
+wget -nc -O model/synset.txt \
+    http://data.mxnet.io/mxnet/models/imagenet/synset.txt
+wget -nc -O model/Inception-BN-0126.params \
+    http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-0126.params?raw=true 
 wget -nc -O cat.jpg https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/python/predict_image/cat.jpg?raw=true
 wget -nc -O dog.jpg https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/python/predict_image/dog.jpg?raw=true
 wget -nc -O model/mean_224.nd https://github.com/dmlc/web-data/raw/master/mxnet/example/feature_extract/mean_224.nd

diff --git a/cpp-package/example/get_data.sh b/cpp-package/example/get_data.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,29 +16,48 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-unameOut="$(uname -s)"
-case "${unameOut}" in
-    Linux*)     CMD='wget';;
-    Darwin*)    CMD='curl -o';;
-    CYGWIN*)    CMD='wget';;
-    MINGW*)     CMD='wget';;
-    *)          CMD=""
-esac
 
-if [ ! -d "./data" ]; then
-    mkdir data
-fi
+set -e
+
+mkdir -p data/mnist_data
+cd data/mnist_data
+
+download () {
+    local URL=$1
+    local GZ_FILE_NAME="${URL##*/}"
+
+    local FILE_NAME="${GZ_FILE_NAME%.*}"
+    if [[ -f "${FILE_NAME}" ]]; then
+        echo "File ${FILE_NAME} already downloaded."
+        return 0
+    fi
 
-if [ ! -d "./data/mnist_data" ]; then
-  mkdir ./data/mnist_data
+    echo "Downloading ${URL} ..."
+    local CURL_OPTIONS="--connect-timeout 10 \
+              --max-time 300 \
+              --retry-delay 10 \
+              --retry 3 \
+              --retry-delay 0 \
+              --location \
+              --silent"
+    curl ${CURL_OPTIONS} ${URL} -o ${GZ_FILE_NAME}
 
-  (cd data/mnist_data; $CMD train-images-idx3-ubyte.gz https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-images-idx3-ubyte.gz)
-  (cd data/mnist_data; $CMD train-labels-idx1-ubyte.gz https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-labels-idx1-ubyte.gz)
-  (cd data/mnist_data; $CMD t10k-images-idx3-ubyte.gz  https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-images-idx3-ubyte.gz)
-  (cd data/mnist_data; $CMD t10k-labels-idx1-ubyte.gz  https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-labels-idx1-ubyte.gz)
-  (cd data/mnist_data; $CMD mnist_train.csv.gz         http://data.mxnet.io/data/mnist_train.csv.gz)
-  (cd data/mnist_data; gzip -d *.gz)
-fi
+    if [[ ! -f "${GZ_FILE_NAME}" ]]; then
+        echo "File ${URL} couldn't be downloaded!"
+        exit 1
+    fi
 
+    gzip -d ${GZ_FILE_NAME}
+    (($? != 0)) && exit 1 || return 0
+}
 
+FILES=(
+    "https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-images-idx3-ubyte.gz"
+    "https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-labels-idx1-ubyte.gz"
+    "https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-images-idx3-ubyte.gz"
+    "https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-labels-idx1-ubyte.gz"
+    "http://data.mxnet.io/data/mnist_train.csv.gz")
 
+for FILE in ${FILES[@]}; do
+    download ${FILE}
+done
diff --git a/cpp-package/example/inference/unit_test_inception_inference.sh b/cpp-package/example/inference/unit_test_inception_inference.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -17,7 +19,7 @@
 
 # Downloading the data and model
 mkdir -p model
-wget -nc http://data.dmlc.ml/mxnet/models/imagenet/inception-bn.tar.gz
+wget -nc http://data.mxnet.io/models/imagenet/inception-bn.tar.gz
 wget -nc -O model/dog.jpg https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/python/predict_image/dog.jpg?raw=true
 wget -nc -O model/mean_224.nd https://github.com/dmlc/web-data/raw/master/mxnet/example/feature_extract/mean_224.nd
 tar -xvzf inception-bn.tar.gz -C model

diff --git a/julia/models/Inception/get.sh b/julia/models/Inception/get.sh
@@ -18,5 +18,5 @@
 # under the License.
 
 
-wget -c http://data.dmlc.ml/mxnet/data/Inception.zip
+wget -c http://data.mxnet.io/mxnet/data/Inception.zip
 unzip Inception.zip
diff --git a/matlab/get_inception_model.sh b/matlab/get_inception_model.sh
@@ -31,5 +31,5 @@ cd ${DATA_DIR}
 wget --no-check-certificate https://raw.githubusercontent.com/dmlc/mxnet.js/master/data/cat.png;
 
 # Get inception model
-wget --no-check-certificate http://data.dmlc.ml/mxnet/models/imagenet/inception-bn.tar.gz;
+wget --no-check-certificate http://data.mxnet.io/models/imagenet/inception-bn.tar.gz
 tar -zxvf inception-bn.tar.gz
diff --git a/tools/dependencies/README.md b/tools/dependencies/README.md
@@ -39,4 +39,4 @@ Currently, we only support gcc-4.8 build. It's your own choice to use a higher v
 This issue appeared in the OSX build with XCode version 8.0 above (reproduced on 9.2). Please add the following build flag in `curl.sh` if your XCode version is more than 8.0:
 ```
 --without-libidn2
-``` 
+``` 
diff --git a/tools/dependencies/curl.sh b/tools/dependencies/curl.sh
@@ -23,7 +23,9 @@ LIBCURL_VERSION=7.61.0
 if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then
     # download and build libcurl
     >&2 echo "Building libcurl..."
-    curl -s -L https://curl.haxx.se/download/curl-$LIBCURL_VERSION.zip -o $DEPS_PATH/libcurl.zip
+    download \
+        https://curl.haxx.se/download/curl-${LIBCURL_VERSION}.zip \
+        ${DEPS_PATH}/libcurl.zip
     unzip -q $DEPS_PATH/libcurl.zip -d $DEPS_PATH
     pushd .
     cd $DEPS_PATH/curl-$LIBCURL_VERSION

diff --git a/tools/dependencies/eigen.sh b/tools/dependencies/eigen.sh
@@ -23,7 +23,9 @@ EIGEN_VERSION=3.3.4
 if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then
     # download eigen
     >&2 echo "Loading eigen..."
-    curl -s -L https://github.com/eigenteam/eigen-git-mirror/archive/$EIGEN_VERSION.zip -o $DEPS_PATH/eigen.zip
+    download \
+        https://github.com/eigenteam/eigen-git-mirror/archive/${EIGEN_VERSION}.zip \
+        ${DEPS_PATH}/eigen.zip
     unzip -q $DEPS_PATH/eigen.zip -d $DEPS_PATH
     mkdir -p $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build
     pushd .

diff --git a/tools/dependencies/libpng.sh b/tools/dependencies/libpng.sh
@@ -23,7 +23,9 @@ PNG_VERSION=1.6.34
 if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then
     # download and build libpng
     >&2 echo "Building libpng..."
-    curl -s -L https://github.com/glennrp/libpng/archive/v$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip
+    download \
+        https://github.com/glennrp/libpng/archive/v${PNG_VERSION}.zip \
+        ${DEPS_PATH}/libpng.zip
     unzip -q $DEPS_PATH/libpng.zip -d $DEPS_PATH
     mkdir -p $DEPS_PATH/libpng-$PNG_VERSION/build
     pushd .

diff --git a/tools/dependencies/libtiff.sh b/tools/dependencies/libtiff.sh
@@ -23,7 +23,9 @@ TIFF_VERSION="4-0-9"
 if [[ ! -f $DEPS_PATH/lib/libtiff.a ]]; then
     # download and build libtiff
     >&2 echo "Building libtiff..."
-    curl -s -L https://gitlab.com/libtiff/libtiff/-/archive/Release-v$TIFF_VERSION/libtiff-Release-v$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip
+    download \
+        https://gitlab.com/libtiff/libtiff/-/archive/Release-v${TIFF_VERSION}/libtiff-Release-v${TIFF_VERSION}.zip \
+        ${DEPS_PATH}/libtiff.zip
     unzip -q $DEPS_PATH/libtiff.zip -d $DEPS_PATH
     pushd .
     cd $DEPS_PATH/libtiff-Release-v$TIFF_VERSION

diff --git a/tools/dependencies/libturbojpeg.sh b/tools/dependencies/libturbojpeg.sh
@@ -28,7 +28,9 @@ fi
 if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a ]]; then
     # download and build libjpeg
     >&2 echo "Building libjpeg-turbo..."
-    curl -s -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/$TURBO_JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip
+    download \
+        https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${TURBO_JPEG_VERSION}.zip \
+        ${DEPS_PATH}/libjpeg.zip
     unzip -q $DEPS_PATH/libjpeg.zip -d $DEPS_PATH
     mkdir -p $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build
     pushd .

diff --git a/tools/dependencies/libz.sh b/tools/dependencies/libz.sh
@@ -23,7 +23,9 @@ ZLIB_VERSION=1.2.6
 if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then
     # Download and build zlib
     >&2 echo "Building zlib..."
-    curl -s -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip
+    download \
+        https://github.com/LuaDist/zlib/archive/${ZLIB_VERSION}.zip \
+        ${DEPS_PATH}/zlib.zip
     unzip -q $DEPS_PATH/zlib.zip -d $DEPS_PATH
     mkdir -p $DEPS_PATH/zlib-$ZLIB_VERSION/build
     pushd .

diff --git a/tools/dependencies/lz4.sh b/tools/dependencies/lz4.sh
@@ -23,7 +23,9 @@ LZ4_VERSION=r130
 if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then
     # Download and build lz4
     >&2 echo "Building lz4..."
-    curl -s -L https://github.com/lz4/lz4/archive/$LZ4_VERSION.zip -o $DEPS_PATH/lz4.zip
+    download \
+        https://github.com/lz4/lz4/archive/${LZ4_VERSION}.zip \
+        ${DEPS_PATH}/lz4.zip
     unzip -q $DEPS_PATH/lz4.zip -d $DEPS_PATH
     pushd .
     cd $DEPS_PATH/lz4-$LZ4_VERSION

diff --git a/tools/dependencies/make_shared_dependencies.sh b/tools/dependencies/make_shared_dependencies.sh
@@ -20,11 +20,37 @@
 # This is a convenience script for calling the build scripts of all dependency libraries.
 # Environment variables should be set beforehand.
 
+set -ex
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
+download () {
+    local URL=$1
+    local OUT_FILE=$2
+
+    if [[ -f "${OUT_FILE}" ]]; then
+        echo "File ${OUT_FILE} already downloaded."
+        return 0
+    fi
+
+    echo "Downloading ${URL} ..."
+    local CURL_OPTIONS="--connect-timeout 10 \
+              --max-time 300 \
+              --retry-delay 10 \
+              --retry 3 \
+              --retry-delay 0 \
+              --location \
+              --silent"
+    curl ${CURL_OPTIONS} ${URL} -o ${OUT_FILE}
+
+    if [[ ! -f "${OUT_FILE}" ]]; then
+        echo "File ${URL} couldn't be downloaded!"
+        exit 1
+    fi
+}
 
 if [[ ! $PLATFORM == 'darwin' ]]; then
-    source $DIR/openblas.sh
+    source ${DIR}/openblas.sh
 fi
 source $DIR/libz.sh
 source $DIR/libturbojpeg.sh

diff --git a/tools/dependencies/openblas.sh b/tools/dependencies/openblas.sh
@@ -25,7 +25,9 @@ if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then
     # download and build openblas
     >&2 echo "Building openblas..."
 
-    curl -s -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip
+    download \
+        https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.zip \
+        ${DEPS_PATH}/openblas.zip
     unzip -q $DEPS_PATH/openblas.zip -d $DEPS_PATH
     pushd .
     cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION

diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh
@@ -39,7 +39,9 @@ fi
 if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopencv_imgcodecs.a ]] || [[ ! -f $DEPS_PATH/lib/libopencv_imgproc.a ]]; then
     # download and build opencv since we need the static library
     >&2 echo "Building opencv..."
-    curl -s -L https://github.com/opencv/opencv/archive/$OPENCV_VERSION.zip -o $DEPS_PATH/opencv.zip
+    download \
+        https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip \
+        ${DEPS_PATH}/opencv.zip
     unzip -q $DEPS_PATH/opencv.zip -d $DEPS_PATH
     mkdir -p $DEPS_PATH/opencv-$OPENCV_VERSION/build
     pushd .

diff --git a/tools/dependencies/openssl.sh b/tools/dependencies/openssl.sh
@@ -24,7 +24,9 @@ if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]];
     # download and build openssl
     >&2 echo "Building openssl..."
     OPENSSL_VERSION=$(echo $OPENSSL_VERSION | sed 's/\./_/g')
-    curl -s -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.zip -o $DEPS_PATH/openssl.zip
+    download \
+        https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION}.zip \
+        ${DEPS_PATH}/openssl.zip
     unzip -q $DEPS_PATH/openssl.zip -d $DEPS_PATH
     pushd .
     cd $DEPS_PATH/openssl-OpenSSL_$OPENSSL_VERSION

diff --git a/tools/dependencies/protobuf.sh b/tools/dependencies/protobuf.sh
@@ -31,7 +31,9 @@ LIBPROTOC="$DEPS_PATH/lib/libprotoc.$DY_EXT"
 if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then
     # Download and build protobuf
     >&2 echo "Building protobuf..."
-    curl -s -L https://github.com/google/protobuf/archive/v$PROTOBUF_VERSION.zip -o $DEPS_PATH/protobuf.zip
+    download \
+        https://github.com/google/protobuf/archive/v${PROTOBUF_VERSION}.zip \
+        ${DEPS_PATH}/protobuf.zip
     unzip -q $DEPS_PATH/protobuf.zip -d $DEPS_PATH
     pushd .
     cd $DEPS_PATH/protobuf-$PROTOBUF_VERSION

diff --git a/tools/dependencies/zmq.sh b/tools/dependencies/zmq.sh
@@ -23,7 +23,9 @@ ZEROMQ_VERSION=4.2.2
 if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
     # Download and build zmq
     >&2 echo "Building zmq..."
-    curl -s -L https://github.com/zeromq/libzmq/archive/v$ZEROMQ_VERSION.zip -o $DEPS_PATH/zeromq.zip
+    download \
+        https://github.com/zeromq/libzmq/archive/v${ZEROMQ_VERSION}.zip \
+        ${DEPS_PATH}/zeromq.zip
     unzip -q $DEPS_PATH/zeromq.zip -d $DEPS_PATH
     mkdir -p $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build
     pushd .