diff --git a/docs/tutorial_utils/data/oxford_102_flower_dataset.py b/docs/tutorial_utils/data/oxford_102_flower_dataset.py new file mode 100644 index 000000000000..0dcae2209c6e --- /dev/null +++ b/docs/tutorial_utils/data/oxford_102_flower_dataset.py @@ -0,0 +1,219 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +""" +This scripts downloads and prepares the Oxford 102 Category Flower Dataset for training +Dataset is from: http://www.robots.ox.ac.uk/~vgg/data/flowers/102/index.html +Script is modified from: https://github.com/Arsey/keras-transfer-learning-for-oxford102 +""" + +import glob +import os +import tarfile +from shutil import copyfile + +import numpy as np +from mxnet import gluon +from scipy.io import loadmat + +label_names = [ + 'pink primrose', + 'hard-leaved pocket orchid', + 'canterbury bells', + 'sweet pea', + 'english marigold', + 'tiger lily', + 'moon orchid', + 'bird of paradise', + 'monkshood', + 'globe thistle', + 'snapdragon', + "colt's foot", + 'king protea', + 'spear thistle', + 'yellow iris', + 'globe-flower', + 'purple coneflower', + 'peruvian lily', + 'balloon flower', + 'giant white arum lily', + 'fire lily', + 'pincushion flower', + 'fritillary', + 'red ginger', + 'grape hyacinth', + 'corn poppy', + 'prince of wales feathers', + 'stemless gentian', + 'artichoke', + 'sweet william', + 'carnation', + 'garden phlox', + 'love in the mist', + 'mexican aster', + 'alpine sea holly', + 'ruby-lipped cattleya', + 'cape flower', + 'great masterwort', + 'siam tulip', + 'lenten rose', + 'barbeton daisy', + 'daffodil', + 'sword lily', + 'poinsettia', + 'bolero deep blue', + 'wallflower', + 'marigold', + 'buttercup', + 'oxeye daisy', + 'common dandelion', + 'petunia', + 'wild pansy', + 'primula', + 'sunflower', + 'pelargonium', + 'bishop of llandaff', + 'gaura', + 'geranium', + 'orange dahlia', + 'pink-yellow dahlia?', + 'cautleya spicata', + 'japanese anemone', + 'black-eyed susan', + 'silverbush', + 'californian poppy', + 'osteospermum', + 'spring crocus', + 'bearded iris', + 'windflower', + 'tree poppy', + 'gazania', + 'azalea', + 'water lily', + 'rose', + 'thorn apple', + 'morning glory', + 'passion flower', + 'lotus', + 'toad lily', + 'anthurium', + 'frangipani', + 'clematis', + 'hibiscus', + 'columbine', + 'desert-rose', + 'tree mallow', + 'magnolia', + 'cyclamen', + 'watercress', + 'canna lily', + 'hippeastrum ', + 'bee balm', + 'ball moss', + 'foxglove', + 'bougainvillea', + 'camellia', + 'mallow', + 'mexican petunia', + 'bromelia', + 'blanket flower', + 'trumpet creeper', + 'blackberry lily' +] + +def download_data(): + data_url = 'http://www.robots.ox.ac.uk/~vgg/data/flowers/102/' + image_file_name = '102flowers.tgz' + label_file_name = 'imagelabels.mat' + setid_file_name = 'setid.mat' + + global data_path, image_path, label_path, setid_path + image_path = os.path.join(data_path, image_file_name) + label_path = os.path.join(data_path, label_file_name) + setid_path = os.path.join(data_path, setid_file_name) + # download the dataset into current directory + if not os.path.exists(data_path): + os.mkdir(data_path) + if not os.path.isfile(image_path): + gluon.utils.download(url=data_url + image_file_name, path=data_path) + if not os.path.exists(os.path.join(data_path, 'jpg')): + print("Extracting downloaded dataset...") + tarfile.open(image_path).extractall(path=data_path) + if not os.path.isfile(label_path): + gluon.utils.download(url=data_url + label_file_name, path=data_path) + if not os.path.isfile(setid_path): + gluon.utils.download(url=data_url + setid_file_name, path=data_path) + + +def prepare_data(): + # Read .mat file containing training, testing, and validation sets. + global data_path, image_path, label_path, setid_path, label_names + setid = loadmat(setid_path) + + idx_train = setid['trnid'][0] - 1 + idx_test = setid['tstid'][0] - 1 + idx_valid = setid['valid'][0] - 1 + + # Read .mat file containing image labels. + image_labels = loadmat(label_path)['labels'][0] + + # Subtract one to get 0-based labels + image_labels -= 1 + + # convert label from number to flower names + image_labels = [label_names[i] for i in image_labels] + # extracted images are stored in folder 'jpg' + files = sorted(glob.glob(os.path.join(data_path, 'jpg', '*.jpg'))) + file_label_pairs = np.array([i for i in zip(files, image_labels)]) + + # move files from extracted folder to train, test, valid + move_files('train', file_label_pairs[idx_test, :]) + move_files('test', file_label_pairs[idx_train, :]) + move_files('valid', file_label_pairs[idx_valid, :]) + + +def move_files(dir_name, file_label_pairs): + data_segment_dir = os.path.join(data_path, dir_name) + if not os.path.exists(data_segment_dir): + os.mkdir(data_segment_dir) + + for label in label_names: + class_dir = os.path.join(data_segment_dir, label) + if not os.path.exists(class_dir): + os.mkdir(class_dir) + + for file, label in file_label_pairs: + src = str(file) + dst = os.path.join(data_path, dir_name, label, src.split(os.sep)[-1]) + copyfile(src, dst) + + +def generate_synset(): + with open('synset.txt', 'w') as f: + # Gluon Dataset API will load synset in sorted order + for label in sorted(label_names): + f.write(label.strip() + '\n') + f.close() + + +def get_data(dir_name): + global data_path + data_path = dir_name + download_data() + prepare_data() + generate_synset() diff --git a/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md b/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md new file mode 100644 index 000000000000..e55e7c97dfee --- /dev/null +++ b/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md @@ -0,0 +1,267 @@ +# MXNet C++ API inference tutorial + +## Overview +MXNet provides various useful tools and interfaces for deploying your model for inference. For example, you can use [MXNet Model Server](https://github.com/awslabs/mxnet-model-server) to start a service and host your trained model easily. +Besides that, you can also use MXNet's different language APIs to integrate your model with your existing service. We provide [Python](https://mxnet.incubator.apache.org/api/python/module/module.html), [Java](https://mxnet.incubator.apache.org/api/java/index.html), [Scala](https://mxnet.incubator.apache.org/api/scala/index.html), and [C++](https://mxnet.incubator.apache.org/api/c++/index.html) APIs. + +This tutorial is a continuation of the [Gluon end to end tutorial](https://github.com/apache/incubator-mxnet/tree/master/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md), we will focus on the MXNet C++ API. We have slightly modified the code in [C++ Inference Example](https://github.com/apache/incubator-mxnet/tree/master/cpp-package/example/inference) for our use case. + +## Prerequisites + +To complete this tutorial, you need: +- Complete the training part of [Gluon end to end tutorial](https://github.com/apache/incubator-mxnet/tree/master/docs/tutorials/gluon/end_to_end_tutorial_training.md) +- Learn the basics about [MXNet C++ API](https://github.com/apache/incubator-mxnet/tree/master/cpp-package) + + +## Setup the MXNet C++ API +To use the C++ API in MXNet, you need to build MXNet from source with C++ package. Please follow the [built from source guide](https://mxnet.incubator.apache.org/install/ubuntu_setup.html), and [C++ Package documentation](https://github.com/apache/incubator-mxnet/tree/master/cpp-package) +to enable the C++ API. +The summary of those two documents is that you need to build MXNet from source with `USE_CPP_PACKAGE` flag set to 1. For example: `make -j USE_CPP_PACKAGE=1`. + +## Load the model and run inference + +After you complete [the previous tutorial](https://github.com/apache/incubator-mxnet/tree/master/docs/tutorials/gluon/end_to_end_tutorial_training.md), you will get the following output files: +1. Model Architecture stored in `flower-recognition-symbol.json` +2. Model parameter values stored in `flower-recognition-0040.params` (`0040` is for 40 epochs we ran) +3. Label names stored in `synset.txt` +4. Mean and standard deviation values stored in `mean_std_224` for image normalization. + + +Now we need to write the C++ code to load them and run prediction on a test image. +The full code is available in the [C++ Inference Example](https://github.com/apache/incubator-mxnet/tree/master/cpp-package/example/inference), we will walk you through it and point out the necessary changes to make for our use case. + + + +### Write a predictor using the MXNet C++ API + +In general, the C++ inference code should follow the 4 steps below. We can do that using a Predictor class. +1. Load the pre-trained model +2. Load the parameters of pre-trained model +3. Load the image to be classified in to NDArray and apply image transformation we did in training +4. Run the forward pass and predict the class of the input image + +```cpp +class Predictor { + public: + Predictor() {} + Predictor(const std::string& model_json_file, + const std::string& model_params_file, + const Shape& input_shape, + bool gpu_context_type = false, + const std::string& synset_file = "", + const std::string& mean_image_file = ""); + void PredictImage(const std::string& image_file); + ~Predictor(); + + private: + void LoadModel(const std::string& model_json_file); + void LoadParameters(const std::string& model_parameters_file); + void LoadSynset(const std::string& synset_file); + NDArray LoadInputImage(const std::string& image_file); + void LoadMeanImageData(); + void LoadDefaultMeanImageData(); + void NormalizeInput(const std::string& mean_image_file); + inline bool FileExists(const std::string& name) { + struct stat buffer; + return (stat(name.c_str(), &buffer) == 0); + } + NDArray mean_img; + std::map args_map; + std::map aux_map; + std::vector output_labels; + Symbol net; + Executor *executor; + Shape input_shape; + NDArray mean_image_data; + NDArray std_dev_image_data; + Context global_ctx = Context::cpu(); + std::string mean_image_file; +}; +``` + +### Load the model, synset file, and normalization values + +In the Predictor constructor, you need to provide paths to saved json and param files. After that, add the following methods `LoadModel` and `LoadParameters` to load the network and its parameters. This part is the same as [the example](https://github.com/apache/incubator-mxnet/blob/master/cpp-package/example/inference/inception_inference.cpp). + +Next, we need to load synset file, and normalization values. We have made the following change since our synset file contains flower names and we used both mean and standard deviation for image normalization. + +```c++ +/* + * The following function loads the synset file. + * This information will be used later to report the label of input image. + */ +void Predictor::LoadSynset(const std::string& synset_file) { + if (!FileExists(synset_file)) { + LG << "Synset file " << synset_file << " does not exist"; + throw std::runtime_error("Synset file does not exist"); + } + LG << "Loading the synset file."; + std::ifstream fi(synset_file.c_str()); + if (!fi.is_open()) { + std::cerr << "Error opening synset file " << synset_file << std::endl; + throw std::runtime_error("Error in opening the synset file."); + } + std::string lemma; + while (getline(fi, lemma)) { + output_labels.push_back(lemma); + } + fi.close(); +} + +/* + * The following function loads the mean and standard deviation values. + * This data will be used for normalizing the image before running the forward + * pass. + * The output data has the same shape as that of the input image data. + */ +void Predictor::LoadMeanImageData() { + LG << "Load the mean image data that will be used to normalize " + << "the image before running forward pass."; + mean_image_data = NDArray(input_shape, global_ctx, false); + mean_image_data.SyncCopyFromCPU( + NDArray::LoadToMap(mean_image_file)["mean_img"].GetData(), + input_shape.Size()); + NDArray::WaitAll(); + std_dev_image_data = NDArray(input_shape, global_ctx, false); + std_dev_image_data.SyncCopyFromCPU( + NDArray::LoadToMap(mean_image_file)["std_img"].GetData(), + input_shape.Size()); + NDArray::WaitAll(); +} +``` + + + +### Load input image + +Now let's add a method to load the input image we want to predict and converts it to NDArray for prediction. +```cpp +NDArray Predictor::LoadInputImage(const std::string& image_file) { + if (!FileExists(image_file)) { + LG << "Image file " << image_file << " does not exist"; + throw std::runtime_error("Image file does not exist"); + } + LG << "Loading the image " << image_file << std::endl; + std::vector array; + cv::Mat mat = cv::imread(image_file); + /*resize pictures to (224, 224) according to the pretrained model*/ + int height = input_shape[2]; + int width = input_shape[3]; + int channels = input_shape[1]; + cv::resize(mat, mat, cv::Size(height, width)); + for (int c = 0; c < channels; ++c) { + for (int i = 0; i < height; ++i) { + for (int j = 0; j < width; ++j) { + array.push_back(static_cast(mat.data[(i * height + j) * 3 + c])); + } + } + } + NDArray image_data = NDArray(input_shape, global_ctx, false); + image_data.SyncCopyFromCPU(array.data(), input_shape.Size()); + NDArray::WaitAll(); + return image_data; +} +``` + +### Predict the image + +Finally, let's run the inference. It's basically using MXNet executor to do a forward pass. To run predictions on multiple images, you can load the images in a list of NDArrays and run prediction in batches. Note that the Predictor class may not be thread safe. Calling it in multi-threaded environments was not tested. To utilize multi-threaded prediction, you need to use the C predict API. Please follow the [C predict example](https://github.com/apache/incubator-mxnet/tree/master/example/image-classification/predict-cpp). + +An additional step is to normalize the image NDArrays values to `(0, 1)` and apply mean and standard deviation we just loaded. + +```cpp +/* + * The following function runs the forward pass on the model. + * The executor is created in the constructor. + * + */ +void Predictor::PredictImage(const std::string& image_file) { + // Load the input image + NDArray image_data = LoadInputImage(image_file); + + // Normalize the image + image_data.Slice(0, 1) /= 255.0; + image_data -= mean_image_data; + image_data /= std_dev_image_data; + + LG << "Running the forward pass on model to predict the image"; + /* + * The executor->arg_arrays represent the arguments to the model. + * + * Copying the image_data that contains the NDArray of input image + * to the arg map of the executor. The input is stored with the key "data" in the map. + * + */ + image_data.CopyTo(&(executor->arg_dict()["data"])); + NDArray::WaitAll(); + + // Run the forward pass. + executor->Forward(false); + + // The output is available in executor->outputs. + auto array = executor->outputs[0].Copy(global_ctx); + NDArray::WaitAll(); + + /* + * Find out the maximum accuracy and the index associated with that accuracy. + * This is done by using the argmax operator on NDArray. + */ + auto predicted = array.ArgmaxChannel(); + NDArray::WaitAll(); + + int best_idx = predicted.At(0, 0); + float best_accuracy = array.At(0, best_idx); + + if (output_labels.empty()) { + LG << "The model predicts the highest accuracy of " << best_accuracy << " at index " + << best_idx; + } else { + LG << "The model predicts the input image to be a [" << output_labels[best_idx] + << " ] with Accuracy = " << best_accuracy << std::endl; + } +} +``` + +### Compile and run the inference code + +You can find the [full code for the inference example](https://github.com/apache/incubator-mxnet/tree/master/cpp-package/example/inference) in the `cpp-package` folder of the project +, and to compile it use this [Makefile](https://github.com/apache/incubator-mxnet/blob/master/cpp-package/example/inference/Makefile). + +Make a copy of the example code, rename it to `flower_inference` and apply the changes we mentioned above. Now you will be able to compile and run inference. Run `make all`. Once this is complete, run inference with the following parameters. Remember to set your `LD_LIBRARY_PATH` to point to MXNet library if you have not done so. + +```bash +make all +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH=:path/to/incubator-mxnet/lib +./flower_inference --symbol flower-recognition-symbol.json --params flower-recognition-0040.params --synset synset.txt --mean mean_std_224.nd --image ./data/test/lotus/image_01832.jpg +``` + +Then it will predict your image: + +```bash +[17:38:51] resnet.cpp:150: Loading the model from flower-recognition-symbol.json + +[17:38:51] resnet.cpp:163: Loading the model parameters from flower-recognition-0040.params + +[17:38:52] resnet.cpp:190: Loading the synset file. +[17:38:52] resnet.cpp:211: Load the mean image data that will be used to normalize the image before running forward pass. +[17:38:52] resnet.cpp:263: Loading the image ./data/test/lotus/image_01832.jpg + +[17:38:52] resnet.cpp:299: Running the forward pass on model to predict the image +[17:38:52] resnet.cpp:331: The model predicts the input image to be a [lotus ] with Accuracy = 8.63046 +``` + + + +## What's next + +Now you can explore more ways to run inference and deploy your models: +1. [Java Inference examples](https://github.com/apache/incubator-mxnet/tree/master/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer) +2. [Scala Inference examples](https://mxnet.incubator.apache.org/tutorials/scala/) +3. [ONNX model inference examples](https://mxnet.incubator.apache.org/tutorials/onnx/inference_on_onnx_model.html) +4. [MXNet Model Server Examples](https://github.com/awslabs/mxnet-model-server/tree/master/examples) + +## References + +1. [Gluon end to end tutorial](https://github.com/apache/incubator-mxnet/tree/master/docs/tutorials/gluon/end_to_end_tutorial_training.md) +2. [Gluon C++ inference example](https://github.com/apache/incubator-mxnet/blob/master/cpp-package/example/inference/) +3. [Gluon C++ package](https://github.com/apache/incubator-mxnet/tree/master/cpp-package) \ No newline at end of file diff --git a/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md b/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md new file mode 100644 index 000000000000..87e6f24ae254 --- /dev/null +++ b/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md @@ -0,0 +1,334 @@ + +# Gluon: from experiment to deployment, an end to end tutorial + +## Overview +MXNet Gluon API comes with a lot of great features, and it can provide you everything you need: from experimentation to deploying the model. In this tutorial, we will walk you through a common use case on how to build a model using gluon, train it on your data, and deploy it for inference. +This tutorial covers training and inference in Python, please continue to [C++ inference part](https://github.com/apache/incubator-mxnet/tree/master/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md) after you finish. + +Let's say you need to build a service that provides flower species recognition. A common problem is that you don't have enough data to train a good model. In such cases, a technique called Transfer Learning can be used to make a more robust model. +In Transfer Learning we make use of a pre-trained model that solves a related task, and was trained on a very large standard dataset, such as ImageNet. ImageNet is from a different domain, but we can utilize the knowledge in this pre-trained model to perform the new task at hand. + +Gluon provides State of the Art models for many of the standard tasks such as Classification, Object Detection, Segmentation, etc. In this tutorial we will use the pre-trained model [ResNet50 V2](https://arxiv.org/abs/1603.05027) trained on ImageNet dataset. This model achieves 77.11% top-1 accuracy on ImageNet. We seek to transfer as much knowledge as possible for our task of recognizing different species of flowers. + + + + +## Prerequisites + +To complete this tutorial, you need: + +- [Build MXNet from source](https://mxnet.incubator.apache.org/install/ubuntu_setup.html#build-mxnet-from-source) with Python(Gluon) and C++ Packages +- Learn the basics about Gluon with [A 60-minute Gluon Crash Course](https://gluon-crash-course.mxnet.io/) + + +## The Data + +We will use the [Oxford 102 Category Flower Dataset](http://www.robots.ox.ac.uk/~vgg/data/flowers/102/) as an example to show you the steps. +We have prepared a utility file to help you download and organize your data into train, test, and validation sets. Run the following Python code to download and prepare the data: + + +```python +import mxnet as mx +data_util_file = "oxford_102_flower_dataset.py" +base_url = "https://raw.githubusercontent.com/roywei/incubator-mxnet/gluon_tutorial/docs/tutorial_utils/data/{}?raw=true" +mx.test_utils.download(base_url.format(data_util_file), fname=data_util_file) +import oxford_102_flower_dataset + +# download and move data to train, test, valid folders +path = './data' +oxford_102_flower_dataset.get_data(path) +``` + +Now your data will be organized into the following format, all the images belong to the same category will be put together in the following pattern: +```bash +data +|--train +| |-- class0 +| | |-- image_06736.jpg +| | |-- image_06741.jpg +... +| |-- class1 +| | |-- image_06755.jpg +| | |-- image_06899.jpg +... +|-- test +| |-- class0 +| | |-- image_00731.jpg +| | |-- image_0002.jpg +... +| |-- class1 +| | |-- image_00036.jpg +| | |-- image_05011.jpg + +``` + +## Training using Gluon + +### Define Hyper-parameters + +Now let's first import necessary packages: + + +```python +import math +import os +import time +from multiprocessing import cpu_count + +from mxnet import autograd +from mxnet import gluon, init +from mxnet.gluon import nn +from mxnet.gluon.data.vision import transforms +from mxnet.gluon.model_zoo.vision import resnet50_v2 +``` + +Next, we define the hyper-parameters that we will use for fine-tuning. We will use the [MXNet learning rate scheduler](https://mxnet.incubator.apache.org/tutorials/gluon/learning_rate_schedules.html) to adjust learning rates during training. + + +```python +classes = 102 +epochs = 40 +lr = 0.001 +per_device_batch_size = 32 +momentum = 0.9 +wd = 0.0001 + +lr_factor = 0.75 +# learning rate change at following epochs +lr_epochs = [10, 20, 30] + +num_gpus = mx.context.num_gpus() +num_workers = cpu_count() +ctx = [mx.gpu(i) for i in range(num_gpus)] if num_gpus > 0 else [mx.cpu()] +batch_size = per_device_batch_size * max(num_gpus, 1) +``` + +Now we will apply data augmentations on training images. This makes minor alterations on the training images, and our model will consider them as distinct images. This can be very useful for fine-tuning on a relatively small dataset, and it will help improve the model. We can use the Gluon [DataSet API](https://mxnet.incubator.apache.org/tutorials/gluon/datasets.html), [DataLoader API](https://mxnet.incubator.apache.org/tutorials/gluon/datasets.html), and [Transform API](https://mxnet.incubator.apache.org/tutorials/gluon/data_augmentation.html) to load the images and apply the following data augmentations: +1. Randomly crop the image and resize it to 224x224 +2. Randomly flip the image horizontally +3. Randomly jitter color and add noise +4. Transpose the data from `[height, width, num_channels]` to `[num_channels, height, width]`, and map values from [0, 255] to [0, 1] +5. Normalize with the mean and standard deviation from the ImageNet dataset. + +For validation and inference, we only need to apply step 1, 4, and 5. We also need to save the mean and standard deviation values for [inference using C++](https://github.com/apache/incubator-mxnet/tree/master/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md). + +```python +jitter_param = 0.4 +lighting_param = 0.1 + +# mean and std for normalizing image value in range (0,1) +mean = [0.485, 0.456, 0.406] +std = [0.229, 0.224, 0.225] + +training_transformer = transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomFlipLeftRight(), + transforms.RandomColorJitter(brightness=jitter_param, contrast=jitter_param, + saturation=jitter_param), + transforms.RandomLighting(lighting_param), + transforms.ToTensor(), + transforms.Normalize(mean, std) +]) + +validation_transformer = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean, std) +]) + +# save mean and std NDArray values for inference +mean_img = mx.nd.stack(*[mx.nd.full((224, 224), m) for m in mean]) +std_img = mx.nd.stack(*[mx.nd.full((224, 224), s) for s in std]) +mx.nd.save('mean_std_224.nd', {"mean_img": mean_img, "std_img": std_img}) + +train_path = os.path.join(path, 'train') +val_path = os.path.join(path, 'valid') +test_path = os.path.join(path, 'test') + +# loading the data and apply pre-processing(transforms) on images +train_data = gluon.data.DataLoader( + gluon.data.vision.ImageFolderDataset(train_path).transform_first(training_transformer), + batch_size=batch_size, shuffle=True, num_workers=num_workers) + +val_data = gluon.data.DataLoader( + gluon.data.vision.ImageFolderDataset(val_path).transform_first(validation_transformer), + batch_size=batch_size, shuffle=False, num_workers=num_workers) + +test_data = gluon.data.DataLoader( + gluon.data.vision.ImageFolderDataset(test_path).transform_first(validation_transformer), + batch_size=batch_size, shuffle=False, num_workers=num_workers) +``` + +### Loading pre-trained model + + +We will use pre-trained ResNet50_v2 model which was pre-trained on the [ImageNet Dataset](http://www.image-net.org/) with 1000 classes. To match the classes in the Flower dataset, we must redefine the last softmax (output) layer to be 102, then initialize the parameters. + +Before we go to training, one unique Gluon feature you should be aware of is hybridization. It allows you to convert your imperative code to a static symbolic graph, which is much more efficient to execute. There are two main benefits of hybridizing your model: better performance and easier serialization for deployment. The best part is that it's as simple as just calling `net.hybridize()`. To know more about Gluon hybridization, please follow the [hybridization tutorial](https://mxnet.incubator.apache.org/tutorials/gluon/hybrid.html). + + + +```python +# load pre-trained resnet50_v2 from model zoo +finetune_net = resnet50_v2(pretrained=True, ctx=ctx) + +# change last softmax layer since number of classes are different +with finetune_net.name_scope(): + finetune_net.output = nn.Dense(classes) +finetune_net.output.initialize(init.Xavier(), ctx=ctx) +# hybridize for better performance +finetune_net.hybridize() + +num_batch = len(train_data) + +# setup learning rate scheduler +iterations_per_epoch = math.ceil(num_batch) +# learning rate change at following steps +lr_steps = [epoch * iterations_per_epoch for epoch in lr_epochs] +schedule = mx.lr_scheduler.MultiFactorScheduler(step=lr_steps, factor=lr_factor, base_lr=lr) + +# setup optimizer with learning rate scheduler, metric, and loss function +sgd_optimizer = mx.optimizer.SGD(learning_rate=lr, lr_scheduler=schedule, momentum=momentum, wd=wd) +metric = mx.metric.Accuracy() +softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss() +``` + +### Fine-tuning model on your custom dataset + +Now let's define the test metrics and start fine-tuning. + + + +```python +def test(net, val_data, ctx): + metric = mx.metric.Accuracy() + for i, (data, label) in enumerate(val_data): + data = gluon.utils.split_and_load(data, ctx_list=ctx, even_split=False) + label = gluon.utils.split_and_load(label, ctx_list=ctx, even_split=False) + outputs = [net(x) for x in data] + metric.update(label, outputs) + return metric.get() + +trainer = gluon.Trainer(finetune_net.collect_params(), optimizer=sgd_optimizer) + +# start with epoch 1 for easier learning rate calculation +for epoch in range(1, epochs + 1): + + tic = time.time() + train_loss = 0 + metric.reset() + + for i, (data, label) in enumerate(train_data): + # get the images and labels + data = gluon.utils.split_and_load(data, ctx_list=ctx, even_split=False) + label = gluon.utils.split_and_load(label, ctx_list=ctx, even_split=False) + with autograd.record(): + outputs = [finetune_net(x) for x in data] + loss = [softmax_cross_entropy(yhat, y) for yhat, y in zip(outputs, label)] + for l in loss: + l.backward() + + trainer.step(batch_size) + train_loss += sum([l.mean().asscalar() for l in loss]) / len(loss) + metric.update(label, outputs) + + _, train_acc = metric.get() + train_loss /= num_batch + _, val_acc = test(finetune_net, val_data, ctx) + + print('[Epoch %d] Train-acc: %.3f, loss: %.3f | Val-acc: %.3f | learning-rate: %.3E | time: %.1f' % + (epoch, train_acc, train_loss, val_acc, trainer.learning_rate, time.time() - tic)) + +_, test_acc = test(finetune_net, test_data, ctx) +print('[Finished] Test-acc: %.3f' % (test_acc)) +``` + +Following is the training result: +```bash +[Epoch 40] Train-acc: 0.945, loss: 0.354 | Val-acc: 0.955 | learning-rate: 4.219E-04 | time: 17.8 +[Finished] Test-acc: 0.952 +``` +In the previous example output, we trained the model using an [AWS p3.8xlarge instance](https://aws.amazon.com/ec2/instance-types/p3/) with 4 Tesla V100 GPUs. We were able to reach a test accuracy of 95.5% with 40 epochs in around 12 minutes. This was really fast because our model was pre-trained on a much larger dataset, ImageNet, with around 1.3 million images. It worked really well to capture features on our small dataset. + + +### Save the fine-tuned model + + +We now have a trained our custom model. This can be serialized into model files using the export function. The export function will export the model architecture into a `.json` file and model parameters into a `.params` file. + + + +```python +finetune_net.export("flower-recognition", epoch=epochs) + +``` + +`export` creates `flower-recognition-symbol.json` and `flower-recognition-0040.params` (`0040` is for 40 epochs we ran) in the current directory. These files can be used for model deployment in the next section. + +## Load the model and run inference using the MXNet Module API + +MXNet provides various useful tools and interfaces for deploying your model for inference. For example, you can use [MXNet Model Server](https://github.com/awslabs/mxnet-model-server) to start a service and host your trained model easily. +Besides that, you can also use MXNet's different language APIs to integrate your model with your existing service. We provide [Python](https://mxnet.incubator.apache.org/api/python/module/module.html), [Java](https://mxnet.incubator.apache.org/api/java/index.html), [Scala](https://mxnet.incubator.apache.org/api/scala/index.html), and [C++](https://mxnet.incubator.apache.org/api/c++/index.html) APIs. + +Here we will briefly introduce how to run inference using Module API in Python. There is more detailed explanation available in the [Predict Image Tutorial](https://mxnet.incubator.apache.org/tutorials/python/predict_image.html). +In general, prediction consists of the following steps: +1. Load the model architecture (symbol file) and trained parameter values (params file) +2. Load the synset file for label names +3. Load the image and apply the same transformation we did on validation dataset during training +4. Run a forward pass on the image data +5. Convert output probabilities to predicted label name + +```python +import numpy as np +from collections import namedtuple + +ctx = mx.cpu() +# load model symbol and params +sym, arg_params, aux_params = mx.model.load_checkpoint('flower-recognition', epochs) +mod = mx.mod.Module(symbol=sym, context=ctx, label_names=None) +mod.bind(for_training=False, data_shapes=[('data', (1, 3, 224, 224))], label_shapes=mod._label_shapes) +mod.set_params(arg_params, aux_params, allow_missing=True) + +# load synset for label names +with open('synset.txt', 'r') as f: + labels = [l.rstrip() for l in f] + +# load an image for prediction +img = mx.image.imread('./data/test/lotus/image_01832.jpg') +# apply transform we did during training +img = validation_transformer(img) +# batchify +img = img.expand_dims(axis=0) +Batch = namedtuple('Batch', ['data']) +mod.forward(Batch([img])) +prob = mod.get_outputs()[0].asnumpy() +prob = np.squeeze(prob) +idx = np.argmax(prob) +print('probability=%f, class=%s' % (prob[idx], labels[idx])) +``` + +Following is the output, you can see the image has been classified as lotus correctly. +```bash +probability=9.798435, class=lotus +``` + +## What's next + +You can continue to the [next tutorial](https://github.com/apache/incubator-mxnet/tree/master/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md) on how to load the model we just trained and run inference using MXNet C++ API. + +You can also find more ways to run inference and deploy your models here: +1. [Java Inference examples](https://github.com/apache/incubator-mxnet/tree/master/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer) +2. [Scala Inference examples](https://mxnet.incubator.apache.org/tutorials/scala/) +3. [ONNX model inference examples](https://mxnet.incubator.apache.org/tutorials/onnx/inference_on_onnx_model.html) +4. [MXNet Model Server Examples](https://github.com/awslabs/mxnet-model-server/tree/master/examples) + +## References + +1. [Transfer Learning for Oxford102 Flower Dataset](https://github.com/Arsey/keras-transfer-learning-for-oxford102) +2. [Gluon book on fine-tuning](https://gluon.mxnet.io/chapter08_computer-vision/fine-tuning.html) +3. [Gluon CV transfer learning tutorial](https://gluon-cv.mxnet.io/build/examples_classification/transfer_learning_minc.html) +4. [Gluon crash course](https://gluon-crash-course.mxnet.io/) +5. [Gluon CPP inference example](https://github.com/apache/incubator-mxnet/blob/master/cpp-package/example/inference/) + + \ No newline at end of file diff --git a/tests/tutorials/test_sanity_tutorials.py b/tests/tutorials/test_sanity_tutorials.py index 644a6118333f..429527db2000 100644 --- a/tests/tutorials/test_sanity_tutorials.py +++ b/tests/tutorials/test_sanity_tutorials.py @@ -28,6 +28,7 @@ 'c++/basics.md', 'c++/index.md', 'c++/subgraphAPI.md', + 'c++/mxnet_cpp_inference_tutorial.md', 'control_flow/index.md', 'embedded/index.md', 'embedded/wine_detector.md', diff --git a/tests/tutorials/test_tutorials.py b/tests/tutorials/test_tutorials.py index 8d8ef398d708..37ba9918fb70 100644 --- a/tests/tutorials/test_tutorials.py +++ b/tests/tutorials/test_tutorials.py @@ -151,6 +151,9 @@ def test_python_logistic_regression() : def test_python_numpy_gotchas() : assert _test_tutorial_nb('gluon/gotchas_numpy_in_mxnet') +def test_gluon_end_to_end(): + assert _test_tutorial_nb('gluon/gluon_from_experiment_to_deployment') + def test_python_mnist(): assert _test_tutorial_nb('python/mnist')