diff --git a/R-package/tests/testthat/get_data.R b/R-package/tests/testthat/get_data.R index 691131c11739..9bcacdb46ac8 100644 --- a/R-package/tests/testthat/get_data.R +++ b/R-package/tests/testthat/get_data.R @@ -55,13 +55,16 @@ GetInception <- function() { if (!dir.exists("model")) { dir.create("model/") } + if (!file.exists("model/Inception-BN-0126.params")) { - download.file("http://data.dmlc.ml/models/imagenet/inception-bn/Inception-BN-0126.params", - destfile = "model/Inception-BN-0126.params") + download.file( + "http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-0126.params?raw=true", + destfile = "model/Inception-BN-0126.params") } if (!file.exists("model/Inception-BN-symbol.json")) { - download.file("http://data.dmlc.ml/models/imagenet/inception-bn/Inception-BN-symbol.json", - destfile = "model/Inception-BN-symbol.json") + download.file( + "http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-symbol.json", + destfile = "model/Inception-BN-symbol.json") } } diff --git a/R-package/vignettes/CatsDogsFinetune.Rmd b/R-package/vignettes/CatsDogsFinetune.Rmd index 680b5a302498..726bb1a43c77 100644 --- a/R-package/vignettes/CatsDogsFinetune.Rmd +++ b/R-package/vignettes/CatsDogsFinetune.Rmd @@ -162,13 +162,13 @@ val <- data$val ## Load pretrained model -Here we use the pretrained model from http://data.dmlc.ml/models/imagenet/. +Here we use the pretrained model from http://data.mxnet.io/mxnet/data/. There are 1000 classes in imagenet, and we need to replace the last fully connected layer with a new layer for 2 classes. ```{r} -download.file('http://data.dmlc.ml/data/Inception.zip', destfile = 'Inception.zip') +download.file('http://data.mxnet.io/mxnet/data/Inception.zip', destfile = 'Inception.zip') unzip("Inception.zip") inception_bn <- mx.model.load("./Inception-BN", iteration = 126) diff --git a/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd b/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd index ff631e0f5ce9..9cfdd5a5473f 100644 --- a/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd +++ b/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd @@ -31,7 +31,7 @@ Make sure you unzip the pre-trained model in current folder. And we can use the loading function to load the model into R. ```{r} -download.file('http://data.dmlc.ml/data/Inception.zip', destfile = 'Inception.zip') +download.file('http://data.mxnet.io/mxnet/data/Inception.zip', destfile = 'Inception.zip') unzip("Inception.zip") model <- mx.model.load("Inception/Inception_BN", iteration = 39) ``` diff --git a/cpp-package/example/feature_extract/run.sh b/cpp-package/example/feature_extract/run.sh index 616445dbd671..b98ddb9eb81e 100755 --- a/cpp-package/example/feature_extract/run.sh +++ b/cpp-package/example/feature_extract/run.sh @@ -17,7 +17,12 @@ # Downloading the data and model mkdir -p model -wget -nc http://data.dmlc.ml/mxnet/models/imagenet/inception-bn.tar.gz +wget -nc -O model/Inception-BN-symbol.json \ + http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-symbol.json +wget -nc -O model/synset.txt \ + http://data.mxnet.io/mxnet/models/imagenet/synset.txt +wget -nc -O model/Inception-BN-0126.params \ + http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-0126.params?raw=true wget -nc -O cat.jpg https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/python/predict_image/cat.jpg?raw=true wget -nc -O dog.jpg https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/python/predict_image/dog.jpg?raw=true wget -nc -O model/mean_224.nd https://github.com/dmlc/web-data/raw/master/mxnet/example/feature_extract/mean_224.nd diff --git a/cpp-package/example/get_data.sh b/cpp-package/example/get_data.sh index 7f975222d0be..f08c8a2902f9 100755 --- a/cpp-package/example/get_data.sh +++ b/cpp-package/example/get_data.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -14,29 +16,34 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -unameOut="$(uname -s)" -case "${unameOut}" in - Linux*) CMD='wget';; - Darwin*) CMD='curl -o';; - CYGWIN*) CMD='wget';; - MINGW*) CMD='wget';; - *) CMD="" -esac -if [ ! -d "./data" ]; then - mkdir data -fi +CURL_OPTIONS='--connect-timeout 5 \ + --max-time 10 \ + --retry 3 \ + --retry-delay 0 \ + --retry-max-time 40 + --location' + +mkdir -p data/mnist_data +cd data/mnist_data -if [ ! -d "./data/mnist_data" ]; then - mkdir ./data/mnist_data +curl ${CURL_OPTIONS} \ + https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-images-idx3-ubyte.gz \ + -o train-images-idx3-ubyte.gz - (cd data/mnist_data; $CMD train-images-idx3-ubyte.gz https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-images-idx3-ubyte.gz) - (cd data/mnist_data; $CMD train-labels-idx1-ubyte.gz https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-labels-idx1-ubyte.gz) - (cd data/mnist_data; $CMD t10k-images-idx3-ubyte.gz https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-images-idx3-ubyte.gz) - (cd data/mnist_data; $CMD t10k-labels-idx1-ubyte.gz https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-labels-idx1-ubyte.gz) - (cd data/mnist_data; $CMD mnist_train.csv.gz http://data.mxnet.io/data/mnist_train.csv.gz) - (cd data/mnist_data; gzip -d *.gz) -fi +curl ${CURL_OPTIONS} \ + https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-labels-idx1-ubyte.gz \ + -o train-labels-idx1-ubyte.gz +curl ${CURL_OPTIONS} \ + https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-images-idx3-ubyte.gz \ + -o t10k-images-idx3-ubyte.gz +curl ${CURL_OPTIONS} \ + https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-labels-idx1-ubyte.gz \ + -o t10k-labels-idx1-ubyte.gz +curl ${CURL_OPTIONS} \ + http://data.mxnet.io/data/mnist_train.csv.gz \ + -o mnist_train.csv.gz +gzip -d *.gz diff --git a/cpp-package/example/inference/unit_test_inception_inference.sh b/cpp-package/example/inference/unit_test_inception_inference.sh index f33b8f19be6d..c3c4630f6e4a 100755 --- a/cpp-package/example/inference/unit_test_inception_inference.sh +++ b/cpp-package/example/inference/unit_test_inception_inference.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,7 +19,7 @@ # Downloading the data and model mkdir -p model -wget -nc http://data.dmlc.ml/mxnet/models/imagenet/inception-bn.tar.gz +wget -nc http://data.mxnet.io/models/imagenet/inception-bn.tar.gz wget -nc -O model/dog.jpg https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/python/predict_image/dog.jpg?raw=true wget -nc -O model/mean_224.nd https://github.com/dmlc/web-data/raw/master/mxnet/example/feature_extract/mean_224.nd tar -xvzf inception-bn.tar.gz -C model diff --git a/julia/models/Inception/get.sh b/julia/models/Inception/get.sh index 16452a361d98..7b7895d65539 100755 --- a/julia/models/Inception/get.sh +++ b/julia/models/Inception/get.sh @@ -18,5 +18,5 @@ # under the License. -wget -c http://data.dmlc.ml/mxnet/data/Inception.zip +wget -c http://data.mxnet.io/mxnet/data/Inception.zip unzip Inception.zip diff --git a/matlab/get_inception_model.sh b/matlab/get_inception_model.sh index af2479b33b83..3c0cb5c4b052 100755 --- a/matlab/get_inception_model.sh +++ b/matlab/get_inception_model.sh @@ -31,5 +31,5 @@ cd ${DATA_DIR} wget --no-check-certificate https://raw.githubusercontent.com/dmlc/mxnet.js/master/data/cat.png; # Get inception model -wget --no-check-certificate http://data.dmlc.ml/mxnet/models/imagenet/inception-bn.tar.gz; +wget --no-check-certificate http://data.mxnet.io/models/imagenet/inception-bn.tar.gz tar -zxvf inception-bn.tar.gz diff --git a/tools/dependencies/README.md b/tools/dependencies/README.md index 1c2f5172a783..081768009db0 100644 --- a/tools/dependencies/README.md +++ b/tools/dependencies/README.md @@ -27,6 +27,7 @@ The scripts use the following environment variables for setting behavior: `DEPS_PATH`: the location in which the libraries are downloaded, built, and installed. `PLATFORM`: name of the OS in lower case. Supported options are 'linux' and 'darwin'. +`CURL_OPTIONS`: Any options for curl downloads to be set (retries, for example). It also expects the following build tools in path: make, cmake, tar, unzip, autoconf, nasm @@ -39,4 +40,4 @@ Currently, we only support gcc-4.8 build. It's your own choice to use a higher v This issue appeared in the OSX build with XCode version 8.0 above (reproduced on 9.2). Please add the following build flag in `curl.sh` if your XCode version is more than 8.0: ``` --without-libidn2 -``` \ No newline at end of file +``` diff --git a/tools/dependencies/curl.sh b/tools/dependencies/curl.sh index bb947715f2ac..9568157eac7e 100755 --- a/tools/dependencies/curl.sh +++ b/tools/dependencies/curl.sh @@ -23,7 +23,8 @@ LIBCURL_VERSION=7.61.0 if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then # download and build libcurl >&2 echo "Building libcurl..." - curl -s -L https://curl.haxx.se/download/curl-$LIBCURL_VERSION.zip -o $DEPS_PATH/libcurl.zip + curl ${CURL_OPTIONS} \ + https://curl.haxx.se/download/curl-${LIBCURL_VERSION}.zip -o ${DEPS_PATH}/libcurl.zip unzip -q $DEPS_PATH/libcurl.zip -d $DEPS_PATH pushd . cd $DEPS_PATH/curl-$LIBCURL_VERSION diff --git a/tools/dependencies/eigen.sh b/tools/dependencies/eigen.sh index a0cd8fcc95ef..ec7a9325009a 100755 --- a/tools/dependencies/eigen.sh +++ b/tools/dependencies/eigen.sh @@ -23,7 +23,9 @@ EIGEN_VERSION=3.3.4 if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then # download eigen >&2 echo "Loading eigen..." - curl -s -L https://github.com/eigenteam/eigen-git-mirror/archive/$EIGEN_VERSION.zip -o $DEPS_PATH/eigen.zip + curl ${CURL_OPTIONS} \ + https://github.com/eigenteam/eigen-git-mirror/archive/${EIGEN_VERSION}.zip \ + -o ${DEPS_PATH}/eigen.zip unzip -q $DEPS_PATH/eigen.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build pushd . diff --git a/tools/dependencies/libpng.sh b/tools/dependencies/libpng.sh index 3faa9f027dcb..f3eaf10d9931 100755 --- a/tools/dependencies/libpng.sh +++ b/tools/dependencies/libpng.sh @@ -23,7 +23,9 @@ PNG_VERSION=1.6.34 if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then # download and build libpng >&2 echo "Building libpng..." - curl -s -L https://github.com/glennrp/libpng/archive/v$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip + curl ${CURL_OPTIONS} \ + https://github.com/glennrp/libpng/archive/v${PNG_VERSION}.zip \ + -o ${DEPS_PATH}/libpng.zip unzip -q $DEPS_PATH/libpng.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/libpng-$PNG_VERSION/build pushd . diff --git a/tools/dependencies/libtiff.sh b/tools/dependencies/libtiff.sh index 2a402bbcac27..6426ab428c36 100755 --- a/tools/dependencies/libtiff.sh +++ b/tools/dependencies/libtiff.sh @@ -23,7 +23,9 @@ TIFF_VERSION="4-0-9" if [[ ! -f $DEPS_PATH/lib/libtiff.a ]]; then # download and build libtiff >&2 echo "Building libtiff..." - curl -s -L https://gitlab.com/libtiff/libtiff/-/archive/Release-v$TIFF_VERSION/libtiff-Release-v$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip + curl ${CURL_OPTIONS} \ + https://gitlab.com/libtiff/libtiff/-/archive/Release-v${TIFF_VERSION}/libtiff-Release-v${TIFF_VERSION}.zip \ + -o ${DEPS_PATH}/libtiff.zip unzip -q $DEPS_PATH/libtiff.zip -d $DEPS_PATH pushd . cd $DEPS_PATH/libtiff-Release-v$TIFF_VERSION diff --git a/tools/dependencies/libturbojpeg.sh b/tools/dependencies/libturbojpeg.sh index ac813ebec1c7..8b6c5026e60a 100755 --- a/tools/dependencies/libturbojpeg.sh +++ b/tools/dependencies/libturbojpeg.sh @@ -28,7 +28,9 @@ fi if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a ]]; then # download and build libjpeg >&2 echo "Building libjpeg-turbo..." - curl -s -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/$TURBO_JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip + curl ${CURL_OPTIONS} \ + https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${TURBO_JPEG_VERSION}.zip \ + -o ${DEPS_PATH}/libjpeg.zip unzip -q $DEPS_PATH/libjpeg.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build pushd . diff --git a/tools/dependencies/libz.sh b/tools/dependencies/libz.sh index c5f9953c8d1c..83143de9b45a 100755 --- a/tools/dependencies/libz.sh +++ b/tools/dependencies/libz.sh @@ -23,7 +23,9 @@ ZLIB_VERSION=1.2.6 if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then # Download and build zlib >&2 echo "Building zlib..." - curl -s -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip + curl ${CURL_OPTIONS} \ + https://github.com/LuaDist/zlib/archive/${ZLIB_VERSION}.zip \ + -o ${DEPS_PATH}/zlib.zip unzip -q $DEPS_PATH/zlib.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/zlib-$ZLIB_VERSION/build pushd . diff --git a/tools/dependencies/lz4.sh b/tools/dependencies/lz4.sh index 478c9925e967..40ba649628a4 100755 --- a/tools/dependencies/lz4.sh +++ b/tools/dependencies/lz4.sh @@ -23,7 +23,9 @@ LZ4_VERSION=r130 if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then # Download and build lz4 >&2 echo "Building lz4..." - curl -s -L https://github.com/lz4/lz4/archive/$LZ4_VERSION.zip -o $DEPS_PATH/lz4.zip + curl ${CURL_OPTIONS} \ + https://github.com/lz4/lz4/archive/${LZ4_VERSION}.zip \ + -o ${DEPS_PATH}/lz4.zip unzip -q $DEPS_PATH/lz4.zip -d $DEPS_PATH pushd . cd $DEPS_PATH/lz4-$LZ4_VERSION diff --git a/tools/dependencies/make_shared_dependencies.sh b/tools/dependencies/make_shared_dependencies.sh index 16508586169a..7eb7e7e4472e 100755 --- a/tools/dependencies/make_shared_dependencies.sh +++ b/tools/dependencies/make_shared_dependencies.sh @@ -20,11 +20,18 @@ # This is a convenience script for calling the build scripts of all dependency libraries. # Environment variables should be set beforehand. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +set -ex +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +CURL_OPTIONS='--connect-timeout 5 \ + --max-time 10 \ + --retry 3 \ + --retry-delay 0 \ + --retry-max-time 40 + --location' if [[ ! $PLATFORM == 'darwin' ]]; then - source $DIR/openblas.sh + source ${DIR}/openblas.sh fi source $DIR/libz.sh source $DIR/libturbojpeg.sh diff --git a/tools/dependencies/openblas.sh b/tools/dependencies/openblas.sh index 27f473919a5b..04ae3f12edc4 100755 --- a/tools/dependencies/openblas.sh +++ b/tools/dependencies/openblas.sh @@ -25,7 +25,9 @@ if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then # download and build openblas >&2 echo "Building openblas..." - curl -s -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip + curl ${CURL_OPTIONS} \ + https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.zip \ + -o ${DEPS_PATH}/openblas.zip unzip -q $DEPS_PATH/openblas.zip -d $DEPS_PATH pushd . cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh index 11c9c2155f91..52bd14dbdae0 100755 --- a/tools/dependencies/opencv.sh +++ b/tools/dependencies/opencv.sh @@ -39,7 +39,8 @@ fi if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopencv_imgcodecs.a ]] || [[ ! -f $DEPS_PATH/lib/libopencv_imgproc.a ]]; then # download and build opencv since we need the static library >&2 echo "Building opencv..." - curl -s -L https://github.com/opencv/opencv/archive/$OPENCV_VERSION.zip -o $DEPS_PATH/opencv.zip + curl ${CURL_OPTIONS} \ + https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip -o ${DEPS_PATH}/opencv.zip unzip -q $DEPS_PATH/opencv.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/opencv-$OPENCV_VERSION/build pushd . diff --git a/tools/dependencies/openssl.sh b/tools/dependencies/openssl.sh index 93284db3e39f..0d2e4c1b8abe 100755 --- a/tools/dependencies/openssl.sh +++ b/tools/dependencies/openssl.sh @@ -24,7 +24,9 @@ if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]]; # download and build openssl >&2 echo "Building openssl..." OPENSSL_VERSION=$(echo $OPENSSL_VERSION | sed 's/\./_/g') - curl -s -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.zip -o $DEPS_PATH/openssl.zip + curl ${CURL_OPTIONS} \ + https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION}.zip \ + -o ${DEPS_PATH}/openssl.zip unzip -q $DEPS_PATH/openssl.zip -d $DEPS_PATH pushd . cd $DEPS_PATH/openssl-OpenSSL_$OPENSSL_VERSION diff --git a/tools/dependencies/protobuf.sh b/tools/dependencies/protobuf.sh index 76ce1de8e822..0a351312e726 100755 --- a/tools/dependencies/protobuf.sh +++ b/tools/dependencies/protobuf.sh @@ -31,7 +31,9 @@ LIBPROTOC="$DEPS_PATH/lib/libprotoc.$DY_EXT" if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then # Download and build protobuf >&2 echo "Building protobuf..." - curl -s -L https://github.com/google/protobuf/archive/v$PROTOBUF_VERSION.zip -o $DEPS_PATH/protobuf.zip + curl ${CURL_OPTIONS} \ + https://github.com/google/protobuf/archive/v${PROTOBUF_VERSION}.zip \ + -o ${DEPS_PATH}/protobuf.zip unzip -q $DEPS_PATH/protobuf.zip -d $DEPS_PATH pushd . cd $DEPS_PATH/protobuf-$PROTOBUF_VERSION diff --git a/tools/dependencies/zmq.sh b/tools/dependencies/zmq.sh index 0042d6bcd073..9cb84d6f2abf 100755 --- a/tools/dependencies/zmq.sh +++ b/tools/dependencies/zmq.sh @@ -23,7 +23,9 @@ ZEROMQ_VERSION=4.2.2 if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then # Download and build zmq >&2 echo "Building zmq..." - curl -s -L https://github.com/zeromq/libzmq/archive/v$ZEROMQ_VERSION.zip -o $DEPS_PATH/zeromq.zip + curl ${CURL_OPTIONS} \ + https://github.com/zeromq/libzmq/archive/v${ZEROMQ_VERSION}.zip \ + -o ${DEPS_PATH}/zeromq.zip unzip -q $DEPS_PATH/zeromq.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build pushd .