Fixes for data links (apache#14526)

* Changed data.dmlc.ml -> data.mxnet.io
* Added retries for downloads
* Removed silent mode for explicit failure reporting

R-package/tests/testthat/get_data.R

@@ -55,13 +55,16 @@ GetInception <- function() {
   if (!dir.exists("model")) {
     dir.create("model/")
   }
+
   if (!file.exists("model/Inception-BN-0126.params")) {
-    download.file("http://data.dmlc.ml/models/imagenet/inception-bn/Inception-BN-0126.params", 
-      destfile = "model/Inception-BN-0126.params")
+    download.file(
+        "http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-0126.params?raw=true",
+        destfile = "model/Inception-BN-0126.params")
   }
   if (!file.exists("model/Inception-BN-symbol.json")) {
-    download.file("http://data.dmlc.ml/models/imagenet/inception-bn/Inception-BN-symbol.json", 
-      destfile = "model/Inception-BN-symbol.json")
+    download.file(
+        "http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-symbol.json",
+        destfile = "model/Inception-BN-symbol.json")
   }
 }
 

R-package/vignettes/CatsDogsFinetune.Rmd

@@ -162,13 +162,13 @@ val   <- data$val
 
 ## Load pretrained model
 
-Here we use the pretrained model from http://data.dmlc.ml/models/imagenet/.
+Here we use the pretrained model from http://data.mxnet.io/mxnet/data/.
 There are 1000 classes in imagenet,
 and we need to replace the last fully connected layer with a new layer for 2 classes.
 
 
 ```{r}
-download.file('http://data.dmlc.ml/data/Inception.zip', destfile = 'Inception.zip')
+download.file('http://data.mxnet.io/mxnet/data/Inception.zip', destfile = 'Inception.zip')
 unzip("Inception.zip")
 inception_bn <- mx.model.load("./Inception-BN", iteration = 126)
 

R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd

@@ -31,7 +31,7 @@ Make sure you unzip the pre-trained model in current folder. And we can use the
 loading function to load the model into R.
 
 ```{r}
-download.file('http://data.dmlc.ml/data/Inception.zip', destfile = 'Inception.zip')
+download.file('http://data.mxnet.io/mxnet/data/Inception.zip', destfile = 'Inception.zip')
 unzip("Inception.zip")
 model <- mx.model.load("Inception/Inception_BN", iteration = 39)
 ```

cpp-package/example/feature_extract/run.sh

@@ -17,7 +17,12 @@
 
 # Downloading the data and model
 mkdir -p model
-wget -nc http://data.dmlc.ml/mxnet/models/imagenet/inception-bn.tar.gz
+wget -nc -O model/Inception-BN-symbol.json \
+    http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-symbol.json
+wget -nc -O model/synset.txt \
+    http://data.mxnet.io/mxnet/models/imagenet/synset.txt
+wget -nc -O model/Inception-BN-0126.params \
+    http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-0126.params?raw=true 
 wget -nc -O cat.jpg https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/python/predict_image/cat.jpg?raw=true
 wget -nc -O dog.jpg https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/python/predict_image/dog.jpg?raw=true
 wget -nc -O model/mean_224.nd https://github.com/dmlc/web-data/raw/master/mxnet/example/feature_extract/mean_224.nd

cpp-package/example/get_data.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,29 +16,48 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-unameOut="$(uname -s)"
-case "${unameOut}" in
-    Linux*)     CMD='wget';;
-    Darwin*)    CMD='curl -o';;
-    CYGWIN*)    CMD='wget';;
-    MINGW*)     CMD='wget';;
-    *)          CMD=""
-esac
 
-if [ ! -d "./data" ]; then
-    mkdir data
-fi
+set -e
+
+mkdir -p data/mnist_data
+cd data/mnist_data
+
+download () {
+    local URL=$1
+    local GZ_FILE_NAME="${URL##*/}"
+
+    local FILE_NAME="${GZ_FILE_NAME%.*}"
+    if [[ -f "${FILE_NAME}" ]]; then
+        echo "File ${FILE_NAME} already downloaded."
+        return 0
+    fi
 
-if [ ! -d "./data/mnist_data" ]; then
-  mkdir ./data/mnist_data
+    echo "Downloading ${URL} ..."
+    local CURL_OPTIONS="--connect-timeout 10 \
+              --max-time 300 \
+              --retry-delay 10 \
+              --retry 3 \
+              --retry-delay 0 \
+              --location \
+              --silent"
+    curl ${CURL_OPTIONS} ${URL} -o ${GZ_FILE_NAME}
 
-  (cd data/mnist_data; $CMD train-images-idx3-ubyte.gz https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-images-idx3-ubyte.gz)
-  (cd data/mnist_data; $CMD train-labels-idx1-ubyte.gz https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-labels-idx1-ubyte.gz)
-  (cd data/mnist_data; $CMD t10k-images-idx3-ubyte.gz  https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-images-idx3-ubyte.gz)
-  (cd data/mnist_data; $CMD t10k-labels-idx1-ubyte.gz  https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-labels-idx1-ubyte.gz)
-  (cd data/mnist_data; $CMD mnist_train.csv.gz         http://data.mxnet.io/data/mnist_train.csv.gz)
-  (cd data/mnist_data; gzip -d *.gz)
-fi
+    if [[ ! -f "${GZ_FILE_NAME}" ]]; then
+        echo "File ${URL} couldn't be downloaded!"
+        exit 1
+    fi
 
+    gzip -d ${GZ_FILE_NAME}
+    (($? != 0)) && exit 1 || return 0
+}
 
+FILES=(
+    "https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-images-idx3-ubyte.gz"
+    "https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/train-labels-idx1-ubyte.gz"
+    "https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-images-idx3-ubyte.gz"
+    "https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/mnist/t10k-labels-idx1-ubyte.gz"
+    "http://data.mxnet.io/data/mnist_train.csv.gz")
 
+for FILE in ${FILES[@]}; do
+    download ${FILE}
+done

julia/models/Inception/get.sh

@@ -18,5 +18,5 @@
 # under the License.
 
 
-wget -c http://data.dmlc.ml/mxnet/data/Inception.zip
+wget -c http://data.mxnet.io/mxnet/data/Inception.zip
 unzip Inception.zip

matlab/get_inception_model.sh

@@ -31,5 +31,5 @@ cd ${DATA_DIR}
 wget --no-check-certificate https://raw.githubusercontent.com/dmlc/mxnet.js/master/data/cat.png;
 
 # Get inception model
-wget --no-check-certificate http://data.dmlc.ml/mxnet/models/imagenet/inception-bn.tar.gz;
+wget --no-check-certificate http://data.mxnet.io/models/imagenet/inception-bn.tar.gz
 tar -zxvf inception-bn.tar.gz