Reworked download scripts

apache · Mar 25, 2019 · 3458655 · 3458655
1 parent d5cfe8d
commit 3458655
Show file tree

Hide file tree

Showing 15 changed files with 91 additions and 42 deletions.
diff --git a/cpp-package/example/get_data.sh b/cpp-package/example/get_data.sh
@@ -17,23 +17,47 @@
 # specific language governing permissions and limitations
 # under the License.
 
-CURL_OPTIONS='--connect-timeout 5 --max-time 10 --retry 3 --retry-delay 0 --retry-max-time 40 -L'
+set -e
 
 mkdir -p data/mnist_data
 cd data/mnist_data
 
-curl ${CURL_OPTIONS} https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-images-idx3-ubyte.gz \
-    -o train-images-idx3-ubyte.gz
-
-curl ${CURL_OPTIONS} https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-labels-idx1-ubyte.gz \
-    -o train-labels-idx1-ubyte.gz
-
-curl ${CURL_OPTIONS} https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-images-idx3-ubyte.gz \
-    -o t10k-images-idx3-ubyte.gz
-
-curl ${CURL_OPTIONS} https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-labels-idx1-ubyte.gz \
-    -o t10k-labels-idx1-ubyte.gz
-
-curl ${CURL_OPTIONS} http://data.mxnet.io/data/mnist_train.csv.gz \
-    -o mnist_train.csv.gz
-gzip -d *.gz
+download () {
+    local URL=$1
+    local GZ_FILE_NAME="${URL##*/}"
+
+    local FILE_NAME="${GZ_FILE_NAME%.*}"
+    if [[ -f "${FILE_NAME}" ]]; then
+        echo "File ${FILE_NAME} already downloaded."
+        return 0
+    fi
+
+    echo "Downloading ${URL} ..."
+    local CURL_OPTIONS="--connect-timeout 10 \
+              --max-time 300 \
+              --retry-delay 10 \
+              --retry 3 \
+              --retry-delay 0 \
+              --location \
+              --silent"
+    curl ${CURL_OPTIONS} ${URL} -o ${GZ_FILE_NAME}
+
+    if [[ ! -f "${GZ_FILE_NAME}" ]]; then
+        echo "File ${URL} couldn't be downloaded!"
+        exit 1
+    fi
+
+    gzip -d ${GZ_FILE_NAME}
+    (($? != 0)) && exit 1 || return 0
+}
+
+FILES=(
+    "https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-images-idx3-ubyte.gz"
+    "https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-labels-idx1-ubyte.gz"
+    "https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-images-idx3-ubyte.gz"
+    "https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-labels-idx1-ubyte.gz"
+    "http://data.mxnet.io/data/mnist_train.csv.gz")
+
+for FILE in ${FILES[@]}; do
+    download ${FILE}
+done
diff --git a/tools/dependencies/README.md b/tools/dependencies/README.md
@@ -27,7 +27,6 @@ The scripts use the following environment variables for setting behavior:
 
 `DEPS_PATH`: the location in which the libraries are downloaded, built, and installed.
 `PLATFORM`: name of the OS in lower case. Supported options are 'linux' and 'darwin'.
-`CURL_OPTIONS`: options for curl downloads to be set (retries, for example).
 
 It also expects the following build tools in path: make, cmake, tar, unzip, autoconf, nasm
 

diff --git a/tools/dependencies/curl.sh b/tools/dependencies/curl.sh
@@ -23,8 +23,9 @@ LIBCURL_VERSION=7.61.0
 if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then
     # download and build libcurl
     >&2 echo "Building libcurl..."
-    curl ${CURL_OPTIONS} \
-        https://curl.haxx.se/download/curl-${LIBCURL_VERSION}.zip -o ${DEPS_PATH}/libcurl.zip
+    download \
+        https://curl.haxx.se/download/curl-${LIBCURL_VERSION}.zip \
+        ${DEPS_PATH}/libcurl.zip
     unzip -q $DEPS_PATH/libcurl.zip -d $DEPS_PATH
     pushd .
     cd $DEPS_PATH/curl-$LIBCURL_VERSION

diff --git a/tools/dependencies/eigen.sh b/tools/dependencies/eigen.sh
@@ -23,9 +23,9 @@ EIGEN_VERSION=3.3.4
 if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then
     # download eigen
     >&2 echo "Loading eigen..."
-    curl ${CURL_OPTIONS} \
+    download \
         https://github.com/eigenteam/eigen-git-mirror/archive/${EIGEN_VERSION}.zip \
-        -o ${DEPS_PATH}/eigen.zip
+        ${DEPS_PATH}/eigen.zip
     unzip -q $DEPS_PATH/eigen.zip -d $DEPS_PATH
     mkdir -p $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build
     pushd .

diff --git a/tools/dependencies/libpng.sh b/tools/dependencies/libpng.sh
@@ -23,9 +23,9 @@ PNG_VERSION=1.6.34
 if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then
     # download and build libpng
     >&2 echo "Building libpng..."
-    curl ${CURL_OPTIONS} \
+    download \
         https://github.com/glennrp/libpng/archive/v${PNG_VERSION}.zip \
-        -o ${DEPS_PATH}/libpng.zip
+        ${DEPS_PATH}/libpng.zip
     unzip -q $DEPS_PATH/libpng.zip -d $DEPS_PATH
     mkdir -p $DEPS_PATH/libpng-$PNG_VERSION/build
     pushd .

diff --git a/tools/dependencies/libtiff.sh b/tools/dependencies/libtiff.sh
@@ -23,9 +23,9 @@ TIFF_VERSION="4-0-9"
 if [[ ! -f $DEPS_PATH/lib/libtiff.a ]]; then
     # download and build libtiff
     >&2 echo "Building libtiff..."
-    curl ${CURL_OPTIONS} \
+    download \
         https://gitlab.com/libtiff/libtiff/-/archive/Release-v${TIFF_VERSION}/libtiff-Release-v${TIFF_VERSION}.zip \
-        -o ${DEPS_PATH}/libtiff.zip
+        ${DEPS_PATH}/libtiff.zip
     unzip -q $DEPS_PATH/libtiff.zip -d $DEPS_PATH
     pushd .
     cd $DEPS_PATH/libtiff-Release-v$TIFF_VERSION

diff --git a/tools/dependencies/libturbojpeg.sh b/tools/dependencies/libturbojpeg.sh
@@ -28,9 +28,9 @@ fi
 if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a ]]; then
     # download and build libjpeg
     >&2 echo "Building libjpeg-turbo..."
-    curl ${CURL_OPTIONS} \
+    download \
         https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${TURBO_JPEG_VERSION}.zip \
-        -o ${DEPS_PATH}/libjpeg.zip
+        ${DEPS_PATH}/libjpeg.zip
     unzip -q $DEPS_PATH/libjpeg.zip -d $DEPS_PATH
     mkdir -p $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build
     pushd .

diff --git a/tools/dependencies/libz.sh b/tools/dependencies/libz.sh
@@ -23,9 +23,9 @@ ZLIB_VERSION=1.2.6
 if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then
     # Download and build zlib
     >&2 echo "Building zlib..."
-    curl ${CURL_OPTIONS} \
+    download \
         https://github.com/LuaDist/zlib/archive/${ZLIB_VERSION}.zip \
-        -o ${DEPS_PATH}/zlib.zip
+        ${DEPS_PATH}/zlib.zip
     unzip -q $DEPS_PATH/zlib.zip -d $DEPS_PATH
     mkdir -p $DEPS_PATH/zlib-$ZLIB_VERSION/build
     pushd .

diff --git a/tools/dependencies/lz4.sh b/tools/dependencies/lz4.sh
@@ -23,9 +23,9 @@ LZ4_VERSION=r130
 if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then
     # Download and build lz4
     >&2 echo "Building lz4..."
-    curl ${CURL_OPTIONS} \
+    download \
         https://github.com/lz4/lz4/archive/${LZ4_VERSION}.zip \
-        -o ${DEPS_PATH}/lz4.zip
+        ${DEPS_PATH}/lz4.zip
     unzip -q $DEPS_PATH/lz4.zip -d $DEPS_PATH
     pushd .
     cd $DEPS_PATH/lz4-$LZ4_VERSION

diff --git a/tools/dependencies/make_shared_dependencies.sh b/tools/dependencies/make_shared_dependencies.sh
@@ -23,7 +23,31 @@
 set -ex
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-CURL_OPTIONS='--connect-timeout 5 --max-time 10 --retry 3 --retry-delay 0 --retry-max-time 40 -L'
+
+download () {
+    local URL=$1
+    local OUT_FILE=$2
+
+    if [[ -f "${OUT_FILE}" ]]; then
+        echo "File ${OUT_FILE} already downloaded."
+        return 0
+    fi
+
+    echo "Downloading ${URL} ..."
+    local CURL_OPTIONS="--connect-timeout 10 \
+              --max-time 300 \
+              --retry-delay 10 \
+              --retry 3 \
+              --retry-delay 0 \
+              --location \
+              --silent"
+    curl ${CURL_OPTIONS} ${URL} -o ${OUT_FILE}
+
+    if [[ ! -f "${OUT_FILE}" ]]; then
+        echo "File ${URL} couldn't be downloaded!"
+        exit 1
+    fi
+}
 
 if [[ ! $PLATFORM == 'darwin' ]]; then
     source ${DIR}/openblas.sh

diff --git a/tools/dependencies/openblas.sh b/tools/dependencies/openblas.sh
@@ -25,9 +25,9 @@ if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then
     # download and build openblas
     >&2 echo "Building openblas..."
 
-    curl ${CURL_OPTIONS} \
+    download \
         https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.zip \
-        -o ${DEPS_PATH}/openblas.zip
+        ${DEPS_PATH}/openblas.zip
     unzip -q $DEPS_PATH/openblas.zip -d $DEPS_PATH
     pushd .
     cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION

diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh
@@ -39,8 +39,9 @@ fi
 if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopencv_imgcodecs.a ]] || [[ ! -f $DEPS_PATH/lib/libopencv_imgproc.a ]]; then
     # download and build opencv since we need the static library
     >&2 echo "Building opencv..."
-    curl ${CURL_OPTIONS} \
-        https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip -o ${DEPS_PATH}/opencv.zip
+    download \
+        https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip \
+        ${DEPS_PATH}/opencv.zip
     unzip -q $DEPS_PATH/opencv.zip -d $DEPS_PATH
     mkdir -p $DEPS_PATH/opencv-$OPENCV_VERSION/build
     pushd .

diff --git a/tools/dependencies/openssl.sh b/tools/dependencies/openssl.sh
@@ -24,9 +24,9 @@ if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]];
     # download and build openssl
     >&2 echo "Building openssl..."
     OPENSSL_VERSION=$(echo $OPENSSL_VERSION | sed 's/\./_/g')
-    curl ${CURL_OPTIONS} \
+    download \
         https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION}.zip \
-        -o ${DEPS_PATH}/openssl.zip
+        ${DEPS_PATH}/openssl.zip
     unzip -q $DEPS_PATH/openssl.zip -d $DEPS_PATH
     pushd .
     cd $DEPS_PATH/openssl-OpenSSL_$OPENSSL_VERSION

diff --git a/tools/dependencies/protobuf.sh b/tools/dependencies/protobuf.sh
@@ -31,9 +31,9 @@ LIBPROTOC="$DEPS_PATH/lib/libprotoc.$DY_EXT"
 if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then
     # Download and build protobuf
     >&2 echo "Building protobuf..."
-    curl ${CURL_OPTIONS} \
+    download \
         https://github.com/google/protobuf/archive/v${PROTOBUF_VERSION}.zip \
-        -o ${DEPS_PATH}/protobuf.zip
+        ${DEPS_PATH}/protobuf.zip
     unzip -q $DEPS_PATH/protobuf.zip -d $DEPS_PATH
     pushd .
     cd $DEPS_PATH/protobuf-$PROTOBUF_VERSION

diff --git a/tools/dependencies/zmq.sh b/tools/dependencies/zmq.sh
@@ -23,9 +23,9 @@ ZEROMQ_VERSION=4.2.2
 if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
     # Download and build zmq
     >&2 echo "Building zmq..."
-    curl ${CURL_OPTIONS} \
+    download \
         https://github.com/zeromq/libzmq/archive/v${ZEROMQ_VERSION}.zip \
-        -o ${DEPS_PATH}/zeromq.zip
+        ${DEPS_PATH}/zeromq.zip
     unzip -q $DEPS_PATH/zeromq.zip -d $DEPS_PATH
     mkdir -p $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build
     pushd .