From af9fd8b30539488e98304cf929bb8ea6f9e9c212 Mon Sep 17 00:00:00 2001
From: Sheng Zha <zhasheng@amazon.com>
Date: Thu, 6 Aug 2020 18:55:32 -0700
Subject: [PATCH] remove data.mxnet.io usage in tests

---
 .../deploy/run-on-aws/use_sagemaker.rst       | 19 +++-------
 docs/static_site/src/pages/api/faq/cloud.md   | 32 ++--------------
 example/gluon/image_classification.py         |  3 +-
 python/mxnet/test_utils.py                    | 20 +++++-----
 tests/nightly/download.sh                     | 38 -------------------
 tests/python/gpu/test_gluon_model_zoo_gpu.py  |  2 +-
 .../test_contrib_gluon_data_vision.py         |  2 +-
 tests/python/unittest/test_gluon_data.py      |  2 +-
 tests/python/unittest/test_image.py           |  2 +-
 .../test_numpy_contrib_gluon_data_vision.py   |  2 +-
 10 files changed, 27 insertions(+), 95 deletions(-)
 delete mode 100644 tests/nightly/download.sh

diff --git a/docs/python_docs/python/tutorials/deploy/run-on-aws/use_sagemaker.rst b/docs/python_docs/python/tutorials/deploy/run-on-aws/use_sagemaker.rst
index d627befbd8fd..dc8052b10cdd 100644
--- a/docs/python_docs/python/tutorials/deploy/run-on-aws/use_sagemaker.rst
+++ b/docs/python_docs/python/tutorials/deploy/run-on-aws/use_sagemaker.rst
@@ -18,7 +18,7 @@
 Run on Amazon SageMaker
 -----------------------
 
-This chapter will give a high level overview about Amazon SageMaker,
+This chapter will give a high level overview about running MXNet on Amazon SageMaker,
 in-depth tutorials can be found on the `Sagemaker
 website <https://docs.aws.amazon.com/sagemaker/latest/dg/whatis.html>`__.
 
@@ -29,16 +29,7 @@ charged by time. Within this notebook you can `fetch, explore and
 prepare training
 data <https://docs.aws.amazon.com/sagemaker/latest/dg/how-it-works-notebooks-instances.html>`__.
 
-::
-
-    import mxnet as mx
-    import sagemaker
-    mx.test_utils.get_cifar10() # Downloads Cifar-10 dataset to ./data
-    sagemaker_session = sagemaker.Session()
-    inputs = sagemaker_session.upload_data(path='data/cifar',
-                                           key_prefix='data/cifar10')
-
-Once the data is ready, you can easily launch training via the SageMaker
+With your own data on the notebook instance, you can easily launch training via the SageMaker
 SDK. So there is no need to manually configure and log into EC2
 instances. You can either bring your own model or use SageMaker's
 `built-in
@@ -51,11 +42,11 @@ instance:
 ::
 
     from sagemaker.mxnet import MXNet as MXNetEstimator
-    estimator = MXNetEstimator(entry_point='train.py', 
+    estimator = MXNetEstimator(entry_point='train.py',
                                role=sagemaker.get_execution_role(),
-                               train_instance_count=1, 
+                               train_instance_count=1,
                                train_instance_type='local',
-                               hyperparameters={'batch_size': 1024, 
+                               hyperparameters={'batch_size': 1024,
                                                 'epochs': 30})
     estimator.fit(inputs)
 
diff --git a/docs/static_site/src/pages/api/faq/cloud.md b/docs/static_site/src/pages/api/faq/cloud.md
index 2a5837b017eb..dd1643cb54a1 100644
--- a/docs/static_site/src/pages/api/faq/cloud.md
+++ b/docs/static_site/src/pages/api/faq/cloud.md
@@ -37,40 +37,16 @@ and maintain the resources for precisely the amount of time needed.
 In this document, we provide a step-by-step guide that will teach you
 how to set up an AWS cluster with _MXNet_. We show how to:
 
-- [Use Amazon S3 to host data](#use-amazon-s3-to-host-data)
-- [Set up an EC2 GPU instance with all dependencies installed](#set-up-an-ec2-gpu-instance)
+- [Use Pre-installed EC2 GPU Instance](#use-pre-installed-ec2-gpu-instance)
 - [Build and run MXNet on a single computer](#build-and-run-mxnet-on-a-gpu-instance)
 - [Set up an EC2 GPU cluster for distributed training](#set-up-an-ec2-gpu-cluster-for-distributed-training)
 
-### Use Amazon S3 to Host Data
-
-Amazon S3 provides distributed data storage which proves especially convenient for hosting large datasets.
-To use S3, you need [AWS credentials](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSGettingStartedGuide/AWSCredentials.html),
-including an `ACCESS_KEY_ID` and a `SECRET_ACCESS_KEY`.
-
-To use _MXNet_ with S3, set the environment variables `AWS_ACCESS_KEY_ID` and
-`AWS_SECRET_ACCESS_KEY` by adding the following two lines in
-`~/.bashrc` (replacing the strings with the correct ones):
-
-```bash
-export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
-export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
-```
-
-There are several ways to upload data to S3. One simple way is to use
-[s3cmd](https://s3tools.org/s3cmd). For example:
-
-```bash
-wget http://data.mxnet.io/mxnet/data/mnist.zip
-unzip mnist.zip && s3cmd put t*-ubyte s3://dmlc/mnist/
-```
-
 ### Use Pre-installed EC2 GPU Instance
 The [Deep Learning AMIs](https://aws.amazon.com/marketplace/search/results?x=0&y=0&searchTerms=Deep+Learning+AMI)
 are a series of images supported and maintained by Amazon Web Services for use
 on Amazon Elastic Compute Cloud (Amazon EC2) and contain the latest MXNet release.
 
-Now you can launch _MXNet_ directly on an EC2 GPU instance.  
+Now you can launch _MXNet_ directly on an EC2 GPU instance.
 You can also use [Jupyter](https://jupyter.org) notebook on EC2 machine.
 Here is a [good tutorial](https://github.com/dmlc/mxnet-notebooks)
 on how to connect to a Jupyter notebook running on an EC2 instance.
@@ -81,7 +57,7 @@ on how to connect to a Jupyter notebook running on an EC2 instance.
 provide a foundational image with NVIDIA CUDA, cuDNN, GPU drivers, Intel
 MKL-DNN, Docker and Nvidia-Docker, etc. for deploying your own custom deep
 learning environment. You may follow the [MXNet Build From Source
-instructions](<https://mxnet.apache.org/get_started/build_from_source easily on
+instructions](https://mxnet.apache.org/get_started/build_from_source) easily on
 the Deep Learning Base AMIs.
 
 ### Set Up an EC2 GPU Cluster for Distributed Training
@@ -146,7 +122,7 @@ Put all of the record files into a folder, and point the data path to the folder
 
 #### Use YARN and SGE
 Although using SSH can be simple when you don't have a cluster scheduling framework,
-_MXNet_ is designed to be portable to various platforms.  
+_MXNet_ is designed to be portable to various platforms.
 We provide scripts available in [tracker](https://github.com/dmlc/dmlc-core/tree/master/tracker)
 to allow running on other cluster frameworks, including Hadoop (YARN) and SGE.
 We welcome contributions from the community of examples of running _MXNet_ on your favorite distributed platform.
diff --git a/example/gluon/image_classification.py b/example/gluon/image_classification.py
index 33583ff20175..2b4d7e8d7216 100644
--- a/example/gluon/image_classification.py
+++ b/example/gluon/image_classification.py
@@ -193,7 +193,8 @@ def train(opt, ctx):
         ctx = [ctx]
 
     train_data, val_data = get_data_iters(dataset, batch_size, opt)
-    net.collect_params().reset_ctx(ctx)
+    for p in net.collect_params().values():
+        p.reset_ctx(ctx)
     trainer = gluon.Trainer(net.collect_params(), 'sgd',
                             optimizer_params={'learning_rate': opt.lr,
                                               'wd': opt.wd,
diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py
index 88bf5d836642..da4f35f8817f 100644
--- a/python/mxnet/test_utils.py
+++ b/python/mxnet/test_utils.py
@@ -1806,9 +1806,11 @@ def get_mnist_ubyte(path='data'):
     if not all(os.path.exists(os.path.join(path, f)) for f in files):
         get_mnist(path)
         for f in files:
-            zip_file_path = os.path.join(path, f) + '.gz'
+            ubyte_file_path = os.path.join(path, f)
+            zip_file_path = ubyte_file_path + '.gz'
             with gzip.GzipFile(zip_file_path) as zf:
-                zf.extractall(path)
+                with open(ubyte_file_path, 'wb') as ubyte_file:
+                    ubyte_file.write(zf.read())
 
 def get_cifar10(path='data'):
     """Downloads CIFAR10 dataset into a directory in the current directory with the name `data`,
@@ -1820,23 +1822,23 @@ def get_cifar10(path='data'):
             (not os.path.exists(os.path.join(path, 'cifar', 'test.rec'))) or \
             (not os.path.exists(os.path.join(path, 'cifar', 'train.lst'))) or \
             (not os.path.exists(os.path.join(path, 'cifar', 'test.lst'))):
-        url = 'http://data.mxnet.io/mxnet/data/cifar10.zip'
+        url = 'https://repo.mxnet.io/gluon/dataset/cifar10/cifar10-b9ac2870.zip'
         sha1 = 'b9ac287012f2dad9dfb49d8271c39ecdd7db376c'
         zip_file_path = mx.gluon.utils.download(url, path=path, sha1_hash=sha1,
                                                 verify_ssl=False)
         with zipfile.ZipFile(zip_file_path) as zf:
             zf.extractall(path)
 
-def get_mnist_iterator(batch_size, input_shape, num_parts=1, part_index=0):
+def get_mnist_iterator(batch_size, input_shape, num_parts=1, part_index=0, path='data'):
     """Returns training and validation iterators for MNIST dataset
     """
 
-    get_mnist_ubyte()
+    get_mnist_ubyte(path)
     flat = len(input_shape) != 3
 
     train_dataiter = mx.io.MNISTIter(
-        image="data/train-images-idx3-ubyte",
-        label="data/train-labels-idx1-ubyte",
+        image=os.path.join(path, "train-images-idx3-ubyte"),
+        label=os.path.join(path, "train-labels-idx1-ubyte"),
         input_shape=input_shape,
         batch_size=batch_size,
         shuffle=True,
@@ -1845,8 +1847,8 @@ def get_mnist_iterator(batch_size, input_shape, num_parts=1, part_index=0):
         part_index=part_index)
 
     val_dataiter = mx.io.MNISTIter(
-        image="data/t10k-images-idx3-ubyte",
-        label="data/t10k-labels-idx1-ubyte",
+        image=os.path.join(path, "t10k-images-idx3-ubyte"),
+        label=os.path.join(path, "t10k-labels-idx1-ubyte"),
         input_shape=input_shape,
         batch_size=batch_size,
         flat=flat,
diff --git a/tests/nightly/download.sh b/tests/nightly/download.sh
deleted file mode 100644
index d07fc6f4ab8a..000000000000
--- a/tests/nightly/download.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-dmlc_download() {
-    url=http://data.mxnet.io/mxnet/datasets/
-    dir=$1
-    file=$2
-    if [ ! -e data/${dir}/$file ]; then
-        wget ${url}/${dir}/${file} -P data/${dir}/ || exit -1
-    else
-        echo "data/${dir}/$file already exits"
-    fi
-}
-
-dmlc_download mnist t10k-images-idx3-ubyte
-dmlc_download mnist t10k-labels-idx1-ubyte
-dmlc_download mnist train-images-idx3-ubyte
-dmlc_download mnist train-labels-idx1-ubyte
-
-dmlc_download cifar10 train.rec
-dmlc_download cifar10 test.rec
diff --git a/tests/python/gpu/test_gluon_model_zoo_gpu.py b/tests/python/gpu/test_gluon_model_zoo_gpu.py
index 1d0d3f4b2313..bad65489d9a1 100644
--- a/tests/python/gpu/test_gluon_model_zoo_gpu.py
+++ b/tests/python/gpu/test_gluon_model_zoo_gpu.py
@@ -36,7 +36,7 @@ def eprint(*args, **kwargs):
 VAL_DATA='data/val-5k-256.rec'
 def download_data():
     return mx.test_utils.download(
-        'http://data.mxnet.io/data/val-5k-256.rec', VAL_DATA)
+        'https://repo.mxnet.io/gluon/dataset/test/val-5k-256-9e70d85e0.rec', VAL_DATA)
 
 @with_seed()
 @pytest.mark.serial
diff --git a/tests/python/unittest/test_contrib_gluon_data_vision.py b/tests/python/unittest/test_contrib_gluon_data_vision.py
index 166b07f843d7..ebbe7062649a 100644
--- a/tests/python/unittest/test_contrib_gluon_data_vision.py
+++ b/tests/python/unittest/test_contrib_gluon_data_vision.py
@@ -51,7 +51,7 @@ def _generate_objects():
 
 
 class TestImage(unittest.TestCase):
-    IMAGES_URL = "http://data.mxnet.io/data/test_images.tar.gz"
+    IMAGES_URL = "https://repo.mxnet.io/gluon/dataset/test/test_images-9cebe48a.tar.gz"
 
     def setUp(self):
         self.IMAGES_DIR = tempfile.mkdtemp()
diff --git a/tests/python/unittest/test_gluon_data.py b/tests/python/unittest/test_gluon_data.py
index b68d03f697fc..abdf558c2b1c 100644
--- a/tests/python/unittest/test_gluon_data.py
+++ b/tests/python/unittest/test_gluon_data.py
@@ -52,7 +52,7 @@ def test_array_dataset():
 def prepare_record(tmpdir_factory):
     test_images = tmpdir_factory.mktemp("test_images")
     test_images_tar = test_images.join("test_images.tar.gz")
-    gluon.utils.download("http://data.mxnet.io/data/test_images.tar.gz", str(test_images_tar))
+    gluon.utils.download("https://repo.mxnet.io/gluon/dataset/test/test_images-9cebe48a.tar.gz", str(test_images_tar))
     tarfile.open(test_images_tar).extractall(str(test_images))
     imgs = os.listdir(str(test_images.join("test_images")))
     record = mx.recordio.MXIndexedRecordIO(str(test_images.join("test.idx")), str(test_images.join("test.rec")), 'w')
diff --git a/tests/python/unittest/test_image.py b/tests/python/unittest/test_image.py
index 9cb287a50b11..d638946ea10a 100644
--- a/tests/python/unittest/test_image.py
+++ b/tests/python/unittest/test_image.py
@@ -110,7 +110,7 @@ def _test_imageiter_last_batch(imageiter_list, assert_data_shape):
 
 
 class TestImage(unittest.TestCase):
-    IMAGES_URL = "http://data.mxnet.io/data/test_images.tar.gz"
+    IMAGES_URL = "https://repo.mxnet.io/gluon/dataset/test/test_images-9cebe48a.tar.gz"
 
     def setUp(self):
         self.IMAGES_DIR = tempfile.mkdtemp()
diff --git a/tests/python/unittest/test_numpy_contrib_gluon_data_vision.py b/tests/python/unittest/test_numpy_contrib_gluon_data_vision.py
index 8c3e76aac91c..6713965fd093 100644
--- a/tests/python/unittest/test_numpy_contrib_gluon_data_vision.py
+++ b/tests/python/unittest/test_numpy_contrib_gluon_data_vision.py
@@ -51,7 +51,7 @@ def _generate_objects():
 
 
 class TestImage(unittest.TestCase):
-    IMAGES_URL = "http://data.mxnet.io/data/test_images.tar.gz"
+    IMAGES_URL = "https://repo.mxnet.io/gluon/dataset/test/test_images-9cebe48a.tar.gz"
 
     def setUp(self):
         self.IMAGES_DIR = tempfile.mkdtemp()