From f90668181dd9cfbadd504cf4a58a21fc764537f1 Mon Sep 17 00:00:00 2001 From: Lai Wei Date: Mon, 11 Feb 2019 15:18:54 -0800 Subject: [PATCH 01/75] increased docker shared memory (#14119) --- tests/nightly/JenkinsfileForBinaries | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index eb003097f61b..53e1c30e188f 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -95,7 +95,7 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/tutorial-test-python2') { utils.unpack_and_init('gpu', mx_lib) - utils.docker_run('ubuntu_nightly_gpu', 'nightly_tutorial_test_ubuntu_python2_gpu', true) + utils.docker_run('ubuntu_nightly_gpu', 'nightly_tutorial_test_ubuntu_python2_gpu', true, '1500m') } } }, @@ -103,7 +103,7 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/tutorial-test-python3') { utils.unpack_and_init('gpu', mx_lib) - utils.docker_run('ubuntu_nightly_gpu', 'nightly_tutorial_test_ubuntu_python3_gpu', true) + utils.docker_run('ubuntu_nightly_gpu', 'nightly_tutorial_test_ubuntu_python3_gpu', true, '1500m') } } } From ab5a0cf6cf87f046d98397edbced251fe6173d6c Mon Sep 17 00:00:00 2001 From: Sandeep Krishnamurthy Date: Mon, 11 Feb 2019 15:44:17 -0800 Subject: [PATCH 02/75] Performance improvement in ToTensor GPU Kernel (#14099) * CPU implementation without Kernel launch/map * Optimal CUDA support for 3D ToTensor operator * Add CUDA kernel for 4D inputs * Fix failing CPU tests for totensor * disable warning on windows * try fix in instance norm windows build failure * Guard omp parallel collapse for windows * Remove warning supression to check if it is ok * fix lint issues * Address code review comments --- src/operator/image/image_random-inl.h | 100 ++++++++++++++++++-------- src/operator/image/image_random.cu | 83 +++++++++++++++++++++ 2 files changed, 153 insertions(+), 30 deletions(-) diff --git a/src/operator/image/image_random-inl.h b/src/operator/image/image_random-inl.h index 448016341f21..392fff4dbf81 100644 --- a/src/operator/image/image_random-inl.h +++ b/src/operator/image/image_random-inl.h @@ -43,8 +43,18 @@ namespace mxnet { namespace op { namespace image { -// There are no parameters for this operator. -// Hence, no arameter registration. +using namespace mshadow; + +#if MXNET_USE_CUDA +// NOTE: Kernel launch/map was extremely costly. +// Hence, we use separate CUDA kernels for these operators. +template +void ToTensorImplCUDA(mshadow::Stream *s, + const T1 input, + const T2 output, + const int req, + const float normalize_factor); +#endif // MXNET_USE_CUDA // Shape and Type inference for image to tensor operator inline bool ToTensorShape(const nnvm::NodeAttrs& attrs, @@ -78,37 +88,39 @@ inline bool ToTensorType(const nnvm::NodeAttrs& attrs, } // Operator Implementation - -template -struct totensor_forward { - template - MSHADOW_XINLINE static void Map(uint32_t c, float* out_data, const DType* in_data, - const int length, const int channel, const int step, - const float normalize_factor = 255.0f) { - #pragma omp parallel for +template +inline void ToTensor(float* out_data, const DType* in_data, + const int length, + const int channels, + const float normalize_factor, + const int step) { + // Microsoft Visual C++ compiler does not support omp collapse + #ifdef _MSC_VER + #pragma omp parallel for + #else + #pragma omp parallel for collapse(2) + #endif // _MSC_VER + for (int c = 0; c < channels; ++c) { for (int i = 0; i < length; ++i) { KERNEL_ASSIGN(out_data[step + c*length + i], req, - (in_data[step + i*channel + c]) / normalize_factor); + (in_data[step + i*channels + c]) / normalize_factor); } } -}; - -template -void ToTensorImpl(const OpContext &ctx, - const std::vector &inputs, - const std::vector &outputs, - const std::vector &req, - const int length, - const uint32_t channel, - const int step = 0) { - mshadow::Stream *s = ctx.get_stream(); +} +inline void ToTensorImpl(const std::vector &inputs, + const std::vector &outputs, + const std::vector &req, + const int length, + const int channel, + const float normalize_factor, + const int step) { MSHADOW_TYPE_SWITCH(inputs[0].type_flag_, DType, { MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, { float* output = outputs[0].dptr(); DType* input = inputs[0].dptr(); - mxnet_op::Kernel, xpu>::Launch( - s, channel, output, input, length, channel, step); + ToTensor(output, input, length, channel, + normalize_factor, step); }); }); } @@ -123,24 +135,52 @@ void ToTensorOpForward(const nnvm::NodeAttrs &attrs, CHECK_EQ(outputs.size(), 1U); CHECK_EQ(req.size(), 1U); + // We do not use temp buffer when performance the operation. + // Hence, this check is necessary. CHECK_EQ(req[0], kWriteTo) << "`to_tensor` does not support inplace updates"; - // 3D Input - (h, w, c) - if (inputs[0].ndim() == 3) { + const float normalize_factor = 255.0f; + + if (std::is_same::value) { + #if MXNET_USE_CUDA + mshadow::Stream *s = ctx.get_stream(); + MSHADOW_TYPE_SWITCH(inputs[0].type_flag_, DType, { + MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, { + if (inputs[0].ndim() == 3) { + Tensor input = inputs[0].get(s); + Tensor output = outputs[0].get(s); + ToTensorImplCUDA, Tensor> + (s, input, output, req_type, normalize_factor); + } else { + Tensor input = inputs[0].get(s); + Tensor output = outputs[0].get(s); + ToTensorImplCUDA, Tensor> + (s, input, output, req_type, normalize_factor); + } + }); + }); + #else + LOG(FATAL) << "Compile with USE_CUDA=1 to use ToTensor operator on GPU."; + #endif // MXNET_USE_CUDA + } else if (inputs[0].ndim() == 3) { + // 3D Input - (h, w, c) const int length = inputs[0].shape_[0] * inputs[0].shape_[1]; - const uint32_t channel = inputs[0].shape_[2]; - ToTensorImpl(ctx, inputs, outputs, req, length, channel); + const int channel = static_cast(inputs[0].shape_[2]); + const int step = 0; + ToTensorImpl(inputs, outputs, req, length, + channel, normalize_factor, step); } else if (inputs[0].ndim() == 4) { // 4D input (n, h, w, c) const int batch_size = inputs[0].shape_[0]; const int length = inputs[0].shape_[1] * inputs[0].shape_[2]; - const uint32_t channel = inputs[0].shape_[3]; + const int channel = static_cast(inputs[0].shape_[3]); const int step = channel * length; #pragma omp parallel for for (auto n = 0; n < batch_size; ++n) { - ToTensorImpl(ctx, inputs, outputs, req, length, channel, n*step); + ToTensorImpl(inputs, outputs, req, length, channel, + normalize_factor, n*step); } } } diff --git a/src/operator/image/image_random.cu b/src/operator/image/image_random.cu index 5f9aff27e85b..6fe53832a89e 100644 --- a/src/operator/image/image_random.cu +++ b/src/operator/image/image_random.cu @@ -21,6 +21,7 @@ * \file image_random.cu * \brief GPU Implementation of image transformation operators */ +#include #include "./image_random-inl.h" #include "../elemwise_op_common.h" @@ -28,6 +29,88 @@ namespace mxnet { namespace op { namespace image { +using namespace mshadow; + +// ToTensor Kernel for 3D input +template +__global__ void ToTensorCudaKernel(const Tensor input, + const Tensor output, + const int req, + const int N, + const int H, + const int W, + const int C, + const float normalize_factor) { + // We process one image per thread block. + // In 3D case, we have only 1 block i.e., blockIdx.x + // We do not use it. + for (int c = 0; c < C; ++c) { + for (int h = threadIdx.y; h < H; h += blockDim.y) { + for (int w = threadIdx.x; w < W; w += blockDim.x) { + KERNEL_ASSIGN(output[c][h][w], req, + input[h][w][c] / normalize_factor); + } + } + } +} + +// ToTensor Kernel for 4D input +template +__global__ void ToTensorCudaKernel(const Tensor input, + const Tensor output, + const int req, + const int N, + const int H, + const int W, + const int C, + const float normalize_factor) { + // We process one image per thread block. + const int n = blockIdx.x; + + for (int c = 0; c < C; ++c) { + for (int h = threadIdx.y; h < H; h += blockDim.y) { + for (int w = threadIdx.x; w < W; w += blockDim.x) { + KERNEL_ASSIGN(output[n][c][h][w], req, + input[n][h][w][c] / normalize_factor); + } + } + } +} + +template +void ToTensorImplCUDA(mshadow::Stream *s, + const T1 input, + const T2 output, + const int req, + const float normalize_factor) { + int blocks, H, W, C, N; + cudaStream_t stream = mshadow::Stream::GetStream(s); + if (std::is_same>::value) { + // 3D Input - (H, W, C) + N = 0; + H = input.size(0); + W = input.size(1); + C = input.size(2); + blocks = 1; + } else { + // 4D Input - (N, H, W, C) + N = input.size(0); + H = input.size(1); + W = input.size(2); + C = input.size(3); + blocks = N > 0 ? N : 1; + blocks = N; + } + // One block per image. + // Number of threads = (32, 32) is optimal, because, + // computation is minimal and overhead of CUDA preparing + // all threads is minimal. + ToTensorCudaKernel + <<>>(input, output, + req, N, H, W, C, normalize_factor); + MSHADOW_CUDA_POST_KERNEL_CHECK(ToTensorCudaKernel); +} + NNVM_REGISTER_OP(_image_to_tensor) .set_attr("FCompute", ToTensorOpForward); From 61d69177479d9be1d79c688a8b7bd7db767c0071 Mon Sep 17 00:00:00 2001 From: Alexander Pivovarov Date: Mon, 11 Feb 2019 16:17:28 -0800 Subject: [PATCH 03/75] Add libhdf5-dev to ubuntu_core.sh (#14079) This package contains header files needed to build and install h5py python package, which is installed by ubuntu_python.sh script using pip. --- ci/docker/install/ubuntu_core.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index fc903e5c8899..3f8ffb726208 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -35,6 +35,7 @@ apt-get install -y \ libatlas-base-dev \ libcurl4-openssl-dev \ libjemalloc-dev \ + libhdf5-dev \ liblapack-dev \ libopenblas-dev \ libopencv-dev \ From f5ba7358d7ff0629f48445cf9dc1ce7fe2fd8e84 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 12 Feb 2019 02:56:49 +0100 Subject: [PATCH 04/75] Addresses comments in runtime feature discovery API (#13964) * Prototype for runtime feature detection * Includes from diamond to quotes * Add CPU feature and BLAS flavour flags * Add BLAS flavour and CPU SSE and AVX flags * MXNET_USE_LAPACK * Fix C++ linting errors * Expose runtime feature detection in the public C API and in the Python API * Refactor Storage -> FeatureSet * Refine documentation * Add failure case * Fix pylint * Address CR comments * Address CR comments * Address CR * Address CR * Address CR * Address CR * remove old files * Fix unit test * Port CMake blas change from #13957 * Fix lint * mxruntime -> libinfo * Fix comments * restore libinfo.py * Rework API for feature detection / libinfo * Refine documentation * Fix lint * Fix lint * Define make_unique only for C++ std < 14 * Add memory include * remove old tests * make_unique fiasco * Fix lint --- CMakeLists.txt | 2 +- include/mxnet/base.h | 9 +- include/mxnet/c_api.h | 14 ++- include/mxnet/{mxfeatures.h => libinfo.h} | 34 +++++- python/mxnet/mxfeatures.py | 103 ------------------ python/mxnet/runtime.py | 48 ++++++++ src/c_api/c_api.cc | 9 +- src/c_api/c_api_profile.cc | 10 +- src/{mxfeatures.cc => libinfo.cc} | 55 +++++++++- tests/cpp/misc/libinfo_test.cc | 33 ++++++ tests/python/unittest/test_features.py | 40 ------- .../{test_libinfo.py => test_runtime.py} | 20 ++-- 12 files changed, 203 insertions(+), 174 deletions(-) rename include/mxnet/{mxfeatures.h => libinfo.h} (82%) delete mode 100644 python/mxnet/mxfeatures.py create mode 100644 python/mxnet/runtime.py rename src/{mxfeatures.cc => libinfo.cc} (75%) create mode 100644 tests/cpp/misc/libinfo_test.cc delete mode 100644 tests/python/unittest/test_features.py rename tests/python/unittest/{test_libinfo.py => test_runtime.py} (71%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f974e8b987c..d8ef524bb389 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -294,8 +294,8 @@ else() add_definitions(-DMXNET_USE_NCCL=0) endif() +include(cmake/ChooseBlas.cmake) if(USE_CUDA AND FIRST_CUDA) - include(cmake/ChooseBlas.cmake) include(3rdparty/mshadow/cmake/Utils.cmake) include(cmake/FirstClassLangCuda.cmake) include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) diff --git a/include/mxnet/base.h b/include/mxnet/base.h index 7f12643119f7..26c1a1bd2b29 100644 --- a/include/mxnet/base.h +++ b/include/mxnet/base.h @@ -35,7 +35,7 @@ #include "nnvm/op.h" #include "nnvm/tuple.h" #include "nnvm/symbolic.h" -#include "mxfeatures.h" +#include "libinfo.h" /*! @@ -403,7 +403,14 @@ template<> struct hash { return res; } }; + +#if __cplusplus < 201402L && !defined(_MSC_VER) +template +inline std::unique_ptr make_unique(Args&&... args) { + return std::unique_ptr(new T(std::forward(args)...)); } +#endif +} // namespace std #include "./tensor_blob.h" //! \endcond diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index d6e13ebcf051..e5e57c10faaa 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -139,6 +139,12 @@ struct MXCallbackList { void **contexts; }; +struct LibFeature { + const char* name; + uint32_t index; + bool enabled; +}; + enum CustomOpCallbacks { kCustomOpDelete, kCustomOpForward, @@ -210,12 +216,12 @@ MXNET_DLL const char *MXGetLastError(); //------------------------------------- /*! - * \brief - * \param feature to check mxfeatures.h - * \param out set to true if the feature is enabled, false otherwise + * \brief Get list of features supported on the runtime + * \param libFeature pointer to array of LibFeature + * \param size of the array * \return 0 when success, -1 when failure happens. */ -MXNET_DLL int MXHasFeature(const mx_uint feature, bool* out); +MXNET_DLL int MXLibInfoFeatures(const struct LibFeature **libFeature, size_t *size); /*! * \brief Seed all global random number generators in mxnet. diff --git a/include/mxnet/mxfeatures.h b/include/mxnet/libinfo.h similarity index 82% rename from include/mxnet/mxfeatures.h rename to include/mxnet/libinfo.h index 10f9b3656692..f35d41a9aa8a 100644 --- a/include/mxnet/mxfeatures.h +++ b/include/mxnet/libinfo.h @@ -18,21 +18,27 @@ */ /*! - * Copyright (c) 2018 by Contributors - * \file mxfeatures.h - * \brief check MXNet features including compile time support + * Copyright (c) 2018 by Contributors + * \file libinfo.h + * \author larroy + * \brief get features of the MXNet library at runtime */ #pragma once +#include +#include +#include +#include #include "dmlc/base.h" #include "mshadow/base.h" +#include "c_api.h" /*! *\brief whether to use opencv support */ #ifndef MXNET_USE_OPENCV -#define MXNET_USE_OPENCV 1 +#define MXNET_USE_OPENCV 0 #endif /*! @@ -124,7 +130,8 @@ namespace features { // Check compile flags such as CMakeLists.txt /// Compile time features -enum : uint32_t { +// ATTENTION: When changing this enum, match the strings in the implementation file! +enum : unsigned { // NVIDIA, CUDA CUDA = 0, CUDNN, @@ -179,10 +186,25 @@ enum : uint32_t { }; +struct EnumNames { + static const std::vector names; +}; + +struct LibInfo { + LibInfo(); + static LibInfo* getInstance(); + const std::array& getFeatures() { + return m_lib_features; + } + private: + std::array m_lib_features; + static std::unique_ptr m_inst; +}; + /*! * \return true if the given feature is supported */ -bool is_enabled(uint32_t feat); +bool is_enabled(unsigned feat); } // namespace features } // namespace mxnet diff --git a/python/mxnet/mxfeatures.py b/python/mxnet/mxfeatures.py deleted file mode 100644 index c546151ab06d..000000000000 --- a/python/mxnet/mxfeatures.py +++ /dev/null @@ -1,103 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# coding: utf-8 -# pylint: disable=not-an-iterable - -"""runtime detection of compile time features in the native library""" - -import ctypes -import enum -from .base import _LIB, check_call, mx_uint - -feature_names = [ - "CUDA", - "CUDNN", - "NCCL", - "CUDA_RTC", - "TENSORRT", - "CPU_SSE", - "CPU_SSE2", - "CPU_SSE3", - "CPU_SSE4_1", - "CPU_SSE4_2", - "CPU_SSE4A", - "CPU_AVX", - "CPU_AVX2", - "OPENMP", - "SSE", - "F16C", - "JEMALLOC", - "BLAS_OPEN", - "BLAS_ATLAS", - "BLAS_MKL", - "BLAS_APPLE", - "LAPACK", - "MKLDNN", - "OPENCV", - "CAFFE", - "PROFILER", - "DIST_KVSTORE", - "CXX14", - "SIGNAL_HANDLER", - "DEBUG" -] - - -Feature = enum.Enum('Feature', {name: index for index, name in enumerate(feature_names)}) - - -def has_feature(feature): - """ - Check the library for compile-time feature at runtime - - Parameters - ---------- - feature : int - An integer representing the feature to check - - Returns - ------- - boolean - True if the feature is enabled, false otherwise - """ - res = ctypes.c_bool() - check_call(_LIB.MXHasFeature(mx_uint(feature), ctypes.byref(res))) - return res.value - - -def features_enabled(): - """ - Returns - ------- - features: list of Feature - list of enabled features in the back-end - """ - res = [] - for f in Feature: - if has_feature(f.value): - res.append(f) - return res - -def features_enabled_str(sep=', '): - """ - Returns - ------- - string with a comma separated list of enabled features in the back-end. For example: - "CPU_SSE, OPENMP, F16C, LAPACK, MKLDNN, OPENCV, SIGNAL_HANDLER, DEBUG" - """ - return sep.join(map(lambda x: x.name, features_enabled())) diff --git a/python/mxnet/runtime.py b/python/mxnet/runtime.py new file mode 100644 index 000000000000..afb393281420 --- /dev/null +++ b/python/mxnet/runtime.py @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# coding: utf-8 +# pylint: disable=not-an-iterable + +"""runtime querying of compile time features in the native library""" + +import ctypes +from .base import _LIB, check_call + +class LibFeature(ctypes.Structure): + """ + Compile time feature description + """ + _fields_ = [ + ("name", ctypes.c_char_p), + ("index", ctypes.c_uint32), + ("enabled", ctypes.c_bool) + ] + +def libinfo_features(): + """ + Check the library for compile-time features. The list of features are maintained in libinfo.h and libinfo.cc + + Returns + ------- + A list of class LibFeature indicating which features are available and enabled + """ + lib_features = ctypes.POINTER(LibFeature)() + lib_features_size = ctypes.c_size_t() + check_call(_LIB.MXLibInfoFeatures(ctypes.byref(lib_features), ctypes.byref(lib_features_size))) + feature_list = [lib_features[i] for i in range(lib_features_size.value)] + return feature_list diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index b436e8ca601b..7e03acccdfae 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -43,7 +43,7 @@ #include "mxnet/kvstore.h" #include "mxnet/rtc.h" #include "mxnet/storage.h" -#include "mxnet/mxfeatures.h" +#include "mxnet/libinfo.h" #include "./c_api_common.h" #include "../operator/custom/custom-inl.h" #include "../operator/tensor/matrix_op-inl.h" @@ -87,9 +87,12 @@ inline int MXAPIGetFunctionRegInfo(const FunRegType *e, // NOTE: return value is added in API_END -int MXHasFeature(const mx_uint feature, bool* out) { +int MXLibInfoFeatures(const struct LibFeature **lib_features, size_t *size) { + using namespace features; API_BEGIN(); - *out = features::is_enabled(feature); + LibInfo* lib_info = LibInfo::getInstance(); + *lib_features = lib_info->getFeatures().data(); + *size = lib_info->getFeatures().size(); API_END(); } diff --git a/src/c_api/c_api_profile.cc b/src/c_api/c_api_profile.cc index dc1b7810147f..0de7b485531c 100644 --- a/src/c_api/c_api_profile.cc +++ b/src/c_api/c_api_profile.cc @@ -52,11 +52,6 @@ struct APICallTimingData { #endif // PROFILE_API_INCLUDE_AS_EVENT }; -template -inline std::unique_ptr make_unique(Args&&... args) { - return std::unique_ptr(new T(std::forward(args)...)); -} - /*! * \brief Per-thread profiling data */ @@ -78,7 +73,7 @@ class ProfilingThreadData { auto iter = tasks_.find(name); if (iter == tasks_.end()) { iter = tasks_.emplace(std::make_pair( - name, make_unique(name, domain))).first; + name, std::make_unique(name, domain))).first; } return iter->second.get(); } @@ -93,7 +88,8 @@ class ProfilingThreadData { // Per-thread so no lock necessary auto iter = events_.find(name); if (iter == events_.end()) { - iter = events_.emplace(std::make_pair(name, make_unique(name))).first; + iter = events_.emplace(std::make_pair(name, + std::make_unique(name))).first; } return iter->second.get(); } diff --git a/src/mxfeatures.cc b/src/libinfo.cc similarity index 75% rename from src/mxfeatures.cc rename to src/libinfo.cc index 7a435d7c81c9..44a834c85b16 100644 --- a/src/mxfeatures.cc +++ b/src/libinfo.cc @@ -19,12 +19,14 @@ /*! * Copyright (c) 2018 by Contributors - * \file mxfeatures.cc + * \file libinfo.cc + * \author larroy * \brief check MXNet features including compile time support */ -#include "mxnet/mxfeatures.h" +#include "mxnet/libinfo.h" #include +#include "mxnet/base.h" namespace mxnet { namespace features { @@ -108,5 +110,54 @@ bool is_enabled(const unsigned feat) { return featureSet.is_enabled(feat); } +LibInfo::LibInfo() { + for (size_t i = 0; i < MAX_FEATURES; ++i) { + m_lib_features[i].name = EnumNames::names[i].c_str(); + m_lib_features[i].enabled = is_enabled(i); + m_lib_features[i].index = i; + } +} + +LibInfo *LibInfo::getInstance() { + if (!m_inst) + m_inst = std::make_unique(); + return m_inst.get(); +} + +std::unique_ptr LibInfo::m_inst = nullptr; + +const std::vector EnumNames::names = { + "CUDA", + "CUDNN", + "NCCL", + "CUDA_RTC", + "TENSORRT", + "CPU_SSE", + "CPU_SSE2", + "CPU_SSE3", + "CPU_SSE4_1", + "CPU_SSE4_2", + "CPU_SSE4A", + "CPU_AVX", + "CPU_AVX2", + "OPENMP", + "SSE", + "F16C", + "JEMALLOC", + "BLAS_OPEN", + "BLAS_ATLAS", + "BLAS_MKL", + "BLAS_APPLE", + "LAPACK", + "MKLDNN", + "OPENCV", + "CAFFE", + "PROFILER", + "DIST_KVSTORE", + "CXX14", + "SIGNAL_HANDLER", + "DEBUG", +}; + } // namespace features } // namespace mxnet diff --git a/tests/cpp/misc/libinfo_test.cc b/tests/cpp/misc/libinfo_test.cc new file mode 100644 index 000000000000..57f8f8d764c3 --- /dev/null +++ b/tests/cpp/misc/libinfo_test.cc @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +using namespace mxnet; +using namespace std; +using namespace mxnet::features; + +/* + * Test that enum and string values are in sync + */ +TEST(RuntimeTest, RuntimeTestAll) { + EXPECT_EQ(EnumNames::names.size(), MAX_FEATURES); + const auto& features = LibInfo::getInstance()->getFeatures(); +} diff --git a/tests/python/unittest/test_features.py b/tests/python/unittest/test_features.py deleted file mode 100644 index ff9118100e41..000000000000 --- a/tests/python/unittest/test_features.py +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import mxnet as mx -import sys -from mxnet.mxfeatures import * -from mxnet.base import MXNetError -from nose.tools import * - -def test_runtime_features(): - for f in Feature: - res = has_feature(f.value) - ok_(type(res) is bool) - for f in features_enabled(): - ok_(type(f) is Feature) - ok_(type(features_enabled_str()) is str) - print("Features enabled: {}".format(features_enabled_str())) - -@raises(MXNetError) -def test_has_feature_2large(): - has_feature(sys.maxsize) - - -if __name__ == "__main__": - import nose - nose.runmodule() diff --git a/tests/python/unittest/test_libinfo.py b/tests/python/unittest/test_runtime.py similarity index 71% rename from tests/python/unittest/test_libinfo.py rename to tests/python/unittest/test_runtime.py index 66bf03111d4b..433301819252 100644 --- a/tests/python/unittest/test_libinfo.py +++ b/tests/python/unittest/test_runtime.py @@ -15,16 +15,22 @@ # specific language governing permissions and limitations # under the License. -import os import mxnet as mx -from mxnet import libinfo +import sys +from mxnet.runtime import * +from mxnet.base import MXNetError +from nose.tools import * -def test_include_path(): - incl_path = libinfo.find_include_path() - assert os.path.exists(incl_path) - assert os.path.isdir(incl_path) +def test_libinfo_features(): + features = libinfo_features() + print("Lib features: ") + for f in features: + print(f.name, f.enabled, f.index) + ok_(type(features) is list) + ok_(len(features) > 0) -if __name__ == '__main__': + +if __name__ == "__main__": import nose nose.runmodule() From 8a63bdecf2d9f12d34fe5874957ae4c867eb5f5b Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 12 Feb 2019 20:24:50 +0800 Subject: [PATCH 05/75] Julia: enable integration test (#14025) --- ci/docker/runtime_functions.sh | 1 + ci/jenkins/Jenkins_steps.groovy | 4 ++-- julia/src/executor.jl | 2 +- julia/test/runtests.jl | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 63a2a1b43983..93f2d8cc058c 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -953,6 +953,7 @@ unittest_ubuntu_cpu_julia() { export PATH="$1/bin:$PATH" export MXNET_HOME='/work/mxnet' export JULIA_DEPOT_PATH='/work/julia-depot' + export INTEGRATION_TEST=1 julia -e 'using InteractiveUtils; versioninfo()' diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 5986eaf37ddc..cfbf484756e5 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1028,7 +1028,7 @@ def test_unix_r_gpu() { def test_unix_julia07_cpu() { return ['Julia 0.7: CPU': { node(NODE_LINUX_CPU) { - ws('workspace/ut-julia07-cpu') { + ws('workspace/ut-it-julia07-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_julia07', false) @@ -1041,7 +1041,7 @@ def test_unix_julia07_cpu() { def test_unix_julia10_cpu() { return ['Julia 1.0: CPU': { node(NODE_LINUX_CPU) { - ws('workspace/ut-julia10-cpu') { + ws('workspace/ut-it-julia10-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_julia10', false) diff --git a/julia/src/executor.jl b/julia/src/executor.jl index 29c21c8f481f..e565617976ce 100644 --- a/julia/src/executor.jl +++ b/julia/src/executor.jl @@ -169,7 +169,7 @@ function simple_bind(self::SymbolicNode, ctx::Context; end end - aux_arrays = [zeros(shape, ctx) for shape in aux_shapes] + aux_arrays = NDArray[zeros(shape, ctx) for shape in aux_shapes] return bind(self, ctx, arg_arrays, args_grad=grad_arrays, grad_req=grad_req, aux_states=aux_arrays) end diff --git a/julia/test/runtests.jl b/julia/test/runtests.jl index 4e5f273950a7..e75df67ad8a4 100644 --- a/julia/test/runtests.jl +++ b/julia/test/runtests.jl @@ -39,7 +39,7 @@ include(joinpath(@__DIR__, "common.jl")) test_dir(joinpath(@__DIR__, "unittest")) # run the basic MNIST mlp example - if haskey(ENV, "CONTINUOUS_INTEGRATION") + if haskey(ENV, "INTEGRATION_TEST") @testset "MNIST Test" begin include(joinpath(BASEDIR, "examples", "mnist", "mlp-test.jl")) end From 489810659bd6897cc8a7053233c4c6ff4e9e73a8 Mon Sep 17 00:00:00 2001 From: Carin Meier Date: Tue, 12 Feb 2019 12:59:11 -0500 Subject: [PATCH 06/75] The latest version of leiningen has a dependency problem with codox (#14132) The document generation tool - this will ping the version to 2.8.3 --- ci/docker/install/ubuntu_clojure.sh | 1 + docs/build_version_doc/setup_docs_ubuntu.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/ci/docker/install/ubuntu_clojure.sh b/ci/docker/install/ubuntu_clojure.sh index 7a94cccae946..b20e35898552 100755 --- a/ci/docker/install/ubuntu_clojure.sh +++ b/ci/docker/install/ubuntu_clojure.sh @@ -27,3 +27,4 @@ echo 'Installing Clojure...' wget https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein chmod 775 lein sudo cp lein /usr/local/bin +echo "Y" | sudo lein downgrade 2.8.3 diff --git a/docs/build_version_doc/setup_docs_ubuntu.sh b/docs/build_version_doc/setup_docs_ubuntu.sh index 22b2fe2c2f71..baa23f684c09 100755 --- a/docs/build_version_doc/setup_docs_ubuntu.sh +++ b/docs/build_version_doc/setup_docs_ubuntu.sh @@ -64,6 +64,7 @@ echo "Installing Clojure dependencies..." wget https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein chmod 775 lein sudo cp lein /usr/local/bin +echo "Y" | sudo lein downgrade 2.8.3 echo "Installing R dependencies..." From ce031daf5e4f343306e01b7b748fed2bad9df685 Mon Sep 17 00:00:00 2001 From: Vandana Kannan Date: Tue, 12 Feb 2019 13:21:29 -0800 Subject: [PATCH 07/75] ONNX export: Support equal length splits (#14121) * ONNX export: Support equal length splits * Fix lint error * Add comment about checking for multiple outputs --- .../contrib/onnx/mx2onnx/_op_translations.py | 6 +++-- .../mxnet/contrib/onnx/mx2onnx/export_onnx.py | 23 +++++++++++-------- .../contrib/onnx/onnx2mx/_op_translations.py | 12 ++++++---- tests/python-pytest/onnx/test_cases.py | 4 ++-- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py index e077824e0226..f9d170d81c13 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py +++ b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py @@ -1537,12 +1537,14 @@ def convert_slice_channel(node, **kwargs): ) return [node] elif squeeze_axis == 0 and num_outputs > 1: + in_shape = kwargs.get('in_shape')[0] + split = in_shape[axis] // num_outputs node = onnx.helper.make_node( "Split", input_nodes, - [name], + [name+'_output'+str(i) for i in range(num_outputs)], axis=axis, - split=[num_outputs], + split=[split for _ in range(num_outputs)], name=name, ) return [node] diff --git a/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py b/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py index d0d4501d89f4..a7b11fc902db 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py +++ b/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py @@ -262,17 +262,20 @@ def create_onnx_graph_proto(self, sym, params, in_shape, in_type, verbose=False) # If converted node is NodeProto, add it in processed nodes list elif isinstance(converted_node, NodeProto): onnx_processed_nodes.append(converted_node) - node_name = converted_node.name if converted_node.name else converted_node.output[0] - if node_name in graph_outputs: - onnx_processed_outputs.append( - make_tensor_value_info( - name=node_name, - elem_type=in_type, - shape=graph_outputs[node_name] + # some operators have multiple outputs, + # therefore, check all output node names + node_names = list(converted_node.output) + for nodename in node_names: + if nodename in graph_outputs: + onnx_processed_outputs.append( + make_tensor_value_info( + name=nodename, + elem_type=in_type, + shape=graph_outputs[nodename] + ) ) - ) - if verbose: - logging.info("Output node is: %s", converted_node.name) + if verbose: + logging.info("Output node is: %s", nodename) elif isinstance(converted_node, TensorProto): raise ValueError("Did not expect TensorProto") else: diff --git a/python/mxnet/contrib/onnx/onnx2mx/_op_translations.py b/python/mxnet/contrib/onnx/onnx2mx/_op_translations.py index dc00feee815b..a7cef7674496 100644 --- a/python/mxnet/contrib/onnx/onnx2mx/_op_translations.py +++ b/python/mxnet/contrib/onnx/onnx2mx/_op_translations.py @@ -484,13 +484,15 @@ def split(attrs, inputs, proto_obj): if not split_list: num_outputs = len(proto_obj.model_metadata.get('output_tensor_data')) else: - raise NotImplementedError("Operator {} in MXNet does not support variable splits." - "Tracking the issue to support variable split here: " - "https://github.com/apache/incubator-mxnet/issues/11594" - .format('split')) + if len(set(split_list)) == 1: + num_outputs = len(split_list) + else: + raise NotImplementedError("Operator {} in MXNet does not support variable splits." + "Tracking the issue to support variable split here: " + "https://github.com/apache/incubator-mxnet/issues/11594" + .format('split')) new_attrs['num_outputs'] = num_outputs - return 'split', new_attrs, inputs def _slice(attrs, inputs, proto_obj): diff --git a/tests/python-pytest/onnx/test_cases.py b/tests/python-pytest/onnx/test_cases.py index b20db23aa1fd..89b60d15e84f 100644 --- a/tests/python-pytest/onnx/test_cases.py +++ b/tests/python-pytest/onnx/test_cases.py @@ -77,7 +77,8 @@ 'test_elu', 'test_max_', 'test_softplus', - 'test_reduce_' + 'test_reduce_', + 'test_split_equal' ], 'import': ['test_gather', 'test_softsign', @@ -88,7 +89,6 @@ 'test_averagepool_2d_precomputed_strides', 'test_averagepool_2d_strides', 'test_averagepool_3d', - 'test_split_equal', 'test_hardmax' ], 'export': ['test_random_uniform', From 87c7addcdd520103e68345768e31ffaffbf66399 Mon Sep 17 00:00:00 2001 From: Lanking Date: Tue, 12 Feb 2019 19:18:12 -0800 Subject: [PATCH 08/75] add Apache header on all XML (#14138) * add Apache header on all XML * add deploy pom --- scala-package/assembly/pom.xml | 16 ++++++++++++++++ .../assembly/src/main/assembly/assembly.xml | 16 ++++++++++++++++ .../assembly/src/main/assembly/javadoc.xml | 16 ++++++++++++++++ .../assembly/src/main/assembly/source.xml | 16 ++++++++++++++++ scala-package/core/pom.xml | 16 ++++++++++++++++ scala-package/deploy/pom.xml | 16 ++++++++++++++++ scala-package/deploy/src/main/deploy/deploy.xml | 16 ++++++++++++++++ scala-package/examples/pom.xml | 16 ++++++++++++++++ scala-package/infer/pom.xml | 16 ++++++++++++++++ scala-package/init-native/pom.xml | 16 ++++++++++++++++ scala-package/init/pom.xml | 16 ++++++++++++++++ scala-package/macros/pom.xml | 16 ++++++++++++++++ scala-package/mxnet-demo/java-demo/pom.xml | 16 ++++++++++++++++ scala-package/mxnet-demo/scala-demo/pom.xml | 16 ++++++++++++++++ scala-package/native/pom.xml | 16 ++++++++++++++++ scala-package/packageTest/core/pom.xml | 16 ++++++++++++++++ scala-package/packageTest/examples/pom.xml | 16 ++++++++++++++++ scala-package/packageTest/infer/pom.xml | 16 ++++++++++++++++ scala-package/packageTest/pom.xml | 16 ++++++++++++++++ scala-package/pom.xml | 16 ++++++++++++++++ scala-package/spark/pom.xml | 16 ++++++++++++++++ .../apache_rat_license_check/rat-excludes | 1 - 22 files changed, 336 insertions(+), 1 deletion(-) diff --git a/scala-package/assembly/pom.xml b/scala-package/assembly/pom.xml index 00aa8682f8fa..b93c181bc1d4 100644 --- a/scala-package/assembly/pom.xml +++ b/scala-package/assembly/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/assembly/src/main/assembly/assembly.xml b/scala-package/assembly/src/main/assembly/assembly.xml index 7525df883dab..eaa30180dbf3 100644 --- a/scala-package/assembly/src/main/assembly/assembly.xml +++ b/scala-package/assembly/src/main/assembly/assembly.xml @@ -1,3 +1,19 @@ + full diff --git a/scala-package/assembly/src/main/assembly/javadoc.xml b/scala-package/assembly/src/main/assembly/javadoc.xml index 8f30a261811c..176fa432190c 100644 --- a/scala-package/assembly/src/main/assembly/javadoc.xml +++ b/scala-package/assembly/src/main/assembly/javadoc.xml @@ -1,3 +1,19 @@ + diff --git a/scala-package/assembly/src/main/assembly/source.xml b/scala-package/assembly/src/main/assembly/source.xml index 87fcde360c48..c06786130192 100644 --- a/scala-package/assembly/src/main/assembly/source.xml +++ b/scala-package/assembly/src/main/assembly/source.xml @@ -1,3 +1,19 @@ + diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 4de65c0b361a..c513d28c53d8 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/deploy/pom.xml b/scala-package/deploy/pom.xml index 4e9da8908be1..74b57077773c 100644 --- a/scala-package/deploy/pom.xml +++ b/scala-package/deploy/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/deploy/src/main/deploy/deploy.xml b/scala-package/deploy/src/main/deploy/deploy.xml index 7bfd20737537..8d248943f2ee 100644 --- a/scala-package/deploy/src/main/deploy/deploy.xml +++ b/scala-package/deploy/src/main/deploy/deploy.xml @@ -1,4 +1,20 @@ + + diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index 565ac6e393a5..ed90d8073675 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/init-native/pom.xml b/scala-package/init-native/pom.xml index 1721f8cbd403..1254f615c64b 100644 --- a/scala-package/init-native/pom.xml +++ b/scala-package/init-native/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml index c514177db17c..fb9e3d3535fb 100644 --- a/scala-package/init/pom.xml +++ b/scala-package/init/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index 52dfde181d72..d6dd0201897f 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/mxnet-demo/java-demo/pom.xml b/scala-package/mxnet-demo/java-demo/pom.xml index 05d04b9e6c14..39253b1ce918 100644 --- a/scala-package/mxnet-demo/java-demo/pom.xml +++ b/scala-package/mxnet-demo/java-demo/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/mxnet-demo/scala-demo/pom.xml b/scala-package/mxnet-demo/scala-demo/pom.xml index 37d60d133c31..b83ba518aa3c 100644 --- a/scala-package/mxnet-demo/scala-demo/pom.xml +++ b/scala-package/mxnet-demo/scala-demo/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/native/pom.xml b/scala-package/native/pom.xml index 7b776d5b5171..cc8c1cc86daf 100644 --- a/scala-package/native/pom.xml +++ b/scala-package/native/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/packageTest/core/pom.xml b/scala-package/packageTest/core/pom.xml index bdcd7662f082..f21c04a3346c 100644 --- a/scala-package/packageTest/core/pom.xml +++ b/scala-package/packageTest/core/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/packageTest/examples/pom.xml b/scala-package/packageTest/examples/pom.xml index 070b78dda99b..dc3500a00181 100644 --- a/scala-package/packageTest/examples/pom.xml +++ b/scala-package/packageTest/examples/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/packageTest/infer/pom.xml b/scala-package/packageTest/infer/pom.xml index 7c5a096d6e14..137d8aed0d10 100644 --- a/scala-package/packageTest/infer/pom.xml +++ b/scala-package/packageTest/infer/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml index f6a16dd77c9d..37b3a9a23931 100644 --- a/scala-package/packageTest/pom.xml +++ b/scala-package/packageTest/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/pom.xml b/scala-package/pom.xml index 5ba6f1f9f498..a782be875be3 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/spark/pom.xml b/scala-package/spark/pom.xml index f2737e9334f4..2acb70b43303 100644 --- a/scala-package/spark/pom.xml +++ b/scala-package/spark/pom.xml @@ -1,4 +1,20 @@ + diff --git a/tests/nightly/apache_rat_license_check/rat-excludes b/tests/nightly/apache_rat_license_check/rat-excludes index 93ac16e42b7d..6e7ae8c1bfdc 100755 --- a/tests/nightly/apache_rat_license_check/rat-excludes +++ b/tests/nightly/apache_rat_license_check/rat-excludes @@ -1,4 +1,3 @@ -.*xml \..* .*css \\.* From 45978a95d91634973003f7143974c615001cb75a Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Wed, 13 Feb 2019 18:15:51 +0800 Subject: [PATCH 09/75] add mkldnn softmax_output (#13699) * add mkldnn softmax_output * fix gpu OP unittest error * fix ci/jenkins/mxnet-validation/unix-gpu compiler error * fix coding style * fix Tao comments * remove blank line, fix indentx * modify according to sandeep's comments * change get CPU engine method, and pravate variable * move macro MXNET_USE_MKLDNN to the head * modify according to Tao's comments * make output layout as input * change API of GetSoftmaxOutputForward * add CommitOutput for mkldnn_softmax_output * trigger Jenkins re-test * add alias Softmax symbol for SoftmaxOutput OP * indent and remove blank line --- src/operator/nn/mkldnn/mkldnn_base-inl.h | 2 + src/operator/nn/mkldnn/mkldnn_ops-inl.h | 10 +- .../nn/mkldnn/mkldnn_softmax_output.cc | 145 ++++++++++++++ src/operator/softmax_output-inl.h | 68 ++++++- src/operator/softmax_output.cc | 181 +++++++++++++++--- src/operator/softmax_output.cu | 14 +- 6 files changed, 380 insertions(+), 40 deletions(-) create mode 100644 src/operator/nn/mkldnn/mkldnn_softmax_output.cc diff --git a/src/operator/nn/mkldnn/mkldnn_base-inl.h b/src/operator/nn/mkldnn/mkldnn_base-inl.h index f770c4aba350..bf220b847c0e 100644 --- a/src/operator/nn/mkldnn/mkldnn_base-inl.h +++ b/src/operator/nn/mkldnn/mkldnn_base-inl.h @@ -174,11 +174,13 @@ struct ActivationParam; struct ConvolutionParam; struct DeconvolutionParam; struct SoftmaxParam; +struct SoftmaxOutputParam; bool SupportMKLDNNAct(const ActivationParam& param); bool SupportMKLDNNAct(const ActivationParam& param, const NDArray &input); bool SupportMKLDNNConv(const ConvolutionParam& params, const NDArray &input); bool SupportMKLDNNDeconv(const DeconvolutionParam& params, const NDArray &input); bool SupportMKLDNNSoftmax(const SoftmaxParam& param); +bool SupportMKLDNNSoftmaxOutput(const SoftmaxOutputParam ¶m); } // namespace op static int GetTypeSize(int dtype) { diff --git a/src/operator/nn/mkldnn/mkldnn_ops-inl.h b/src/operator/nn/mkldnn/mkldnn_ops-inl.h index 50937706d934..39f26325b2a5 100644 --- a/src/operator/nn/mkldnn/mkldnn_ops-inl.h +++ b/src/operator/nn/mkldnn/mkldnn_ops-inl.h @@ -76,6 +76,12 @@ void MKLDNNSoftmaxForward(const nnvm::NodeAttrs& attrs, const OpContext &ctx, const NDArray &in_data, const OpReqType &req, const NDArray &out_data); +/* For softmax_output */ +void MKLDNNSoftmaxOutputForward(const nnvm::NodeAttrs& attrs, const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data); + /* For sum */ void MKLDNNSumForward(const nnvm::NodeAttrs& attrs, const OpContext &ctx, const std::vector &inputs, const OpReqType &req, @@ -83,8 +89,8 @@ void MKLDNNSumForward(const nnvm::NodeAttrs& attrs, const OpContext &ctx, /* For copy */ void MKLDNNCopy(const nnvm::NodeAttrs& attrs, const OpContext &ctx, - const NDArray &in_data, const OpReqType &req, - const NDArray &out_data); + const NDArray &in_data, const OpReqType &req, + const NDArray &out_data); /* For concat */ void MKLDNNConcatForward(const nnvm::NodeAttrs& attrs, const OpContext &ctx, diff --git a/src/operator/nn/mkldnn/mkldnn_softmax_output.cc b/src/operator/nn/mkldnn/mkldnn_softmax_output.cc new file mode 100644 index 000000000000..ae34fe633d6f --- /dev/null +++ b/src/operator/nn/mkldnn/mkldnn_softmax_output.cc @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file mkldnn_softmax_output.cc + * \brief integrate mkldnn softmax to softmax_output forward + * \author Zhang Rong A +*/ + +#if MXNET_USE_MKLDNN == 1 +#include "../../softmax_output-inl.h" +#include "./mkldnn_ops-inl.h" +#include "./mkldnn_base-inl.h" + +namespace mxnet { +namespace op { + +static mkldnn::softmax_forward::primitive_desc GetSoftmaxOutputFwdDescImpl( + const SoftmaxOutputParam& param, bool is_train, + const int axis, const mkldnn::memory &input_mem) { + mkldnn::memory::primitive_desc data_mpd = input_mem.get_primitive_desc(); + mkldnn::memory::desc data_md = data_mpd.desc(); + auto cpu_engine = CpuEngine::Get()->get_engine(); + auto prop = is_train ? mkldnn::prop_kind::forward_training + : mkldnn::prop_kind::forward_scoring; + auto desc = mkldnn::softmax_forward::desc(prop, data_md, axis); + return mkldnn::softmax_forward::primitive_desc(desc, cpu_engine); +} + +typedef ParamOpSign MKLDNNSoftmaxOuputSignature; + +class MKLDNNSoftmaxOutputFwd { + std::shared_ptr fwd_; + std::shared_ptr data_; + std::shared_ptr out_; + + public: + const mkldnn::softmax_forward::primitive_desc fwd_pd; + + MKLDNNSoftmaxOutputFwd(const SoftmaxOutputParam& param, bool is_train, + const int axis, const mkldnn::memory &mem): fwd_pd( + GetSoftmaxOutputFwdDescImpl(param, is_train, axis, mem)) { + } + + void SetNewMem(const mkldnn::memory &data, const mkldnn::memory &output) { + if (this->data_ == nullptr) + this->data_ = std::shared_ptr(new mkldnn::memory( + data.get_primitive_desc(), data.get_data_handle())); + else + this->data_->set_data_handle(data.get_data_handle()); + + if (this->out_ == nullptr) + this->out_ = std::shared_ptr(new mkldnn::memory( + output.get_primitive_desc(), output.get_data_handle())); + else + this->out_->set_data_handle(output.get_data_handle()); + + if (this->fwd_ == nullptr) { + this->fwd_ = std::shared_ptr( + new mkldnn::softmax_forward(fwd_pd, mkldnn::primitive::at(*this->data_), + *this->out_)); + } + } + + const mkldnn::softmax_forward &GetFwd() const { + return *fwd_; + } +}; + +static MKLDNNSoftmaxOutputFwd &GetSoftmaxOutputForward(const SoftmaxOutputParam& param, + const OpContext &ctx, + const NDArray &in_data) { +#if DMLC_CXX11_THREAD_LOCAL + static thread_local + std::unordered_map fwds; +#else + static MX_THREAD_LOCAL + std::unordered_map fwds; +#endif + MKLDNNSoftmaxOuputSignature key(param); + key.AddSign(ctx.is_train); + key.AddSign(in_data); + + // softmax_output has no axis parameter, so use it as it original implement. + int axis = in_data.shape().ndim() - 1; + + auto it = fwds.find(key); + if (it == fwds.end()) { + auto in_mem = *(in_data.GetMKLDNNData()); + MKLDNNSoftmaxOutputFwd fwd(param, ctx.is_train, axis, in_mem); + it = AddToCache(&fwds, key, fwd); + } + return it->second; +} + +// This is only used for forward. For backward ,need double check compatibility +bool SupportMKLDNNSoftmaxOutput(const SoftmaxOutputParam ¶m) { + return param.multi_output ? false : true; +} + +void MKLDNNSoftmaxOutputForward(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data) { + const SoftmaxOutputParam ¶m = nnvm::get(attrs.parsed); + + NDArray idata = in_data[softmaxout_enum::kData]; + NDArray odata = out_data[softmaxout_enum::kOut]; + if (in_data[softmaxout_enum::kData].IsView() && in_data[softmaxout_enum::kData].IsMKLDNNData()) { + idata = in_data[softmaxout_enum::kData].Reorder2Default(); + } + + auto input_mem = idata.GetMKLDNNData(); + auto out_mem = CreateMKLDNNMem(out_data[softmaxout_enum::kOut], + input_mem->get_primitive_desc(), req[softmaxout_enum::kOut]); + + MKLDNNSoftmaxOutputFwd &fwd = GetSoftmaxOutputForward(param, ctx, idata); + fwd.SetNewMem(*input_mem, *out_mem.second); + + MKLDNNStream *stream = MKLDNNStream::Get(); + stream->RegisterPrim(fwd.GetFwd()); + + CommitOutput(out_data[softmaxout_enum::kOut], out_mem); + stream->Submit(); +} +} // namespace op +} // namespace mxnet +#endif diff --git a/src/operator/softmax_output-inl.h b/src/operator/softmax_output-inl.h index fec321b97e4c..5a01d3a73a95 100644 --- a/src/operator/softmax_output-inl.h +++ b/src/operator/softmax_output-inl.h @@ -88,6 +88,17 @@ struct SoftmaxOutputParam : public dmlc::Parameter { "one-hot encoding of the gold label and distributed uniformly to" "all other labels."); }; + + bool operator==(const SoftmaxOutputParam& other) const { + return this->grad_scale == other.grad_scale && + this->ignore_label == other.ignore_label && + this->multi_output == other.multi_output && + this->use_ignore == other.use_ignore && + this->preserve_shape == other.preserve_shape && + this->normalization == other.normalization && + this->out_grad == other.out_grad && + this->smooth_alpha == other.smooth_alpha; + } }; template @@ -267,9 +278,43 @@ class SoftmaxOutputOp : public Operator { SoftmaxOutputParam param_; }; // class SoftmaxOutputOp -// Decalre Factory function, used for dispatch specialization template -Operator* CreateOp(SoftmaxOutputParam param, int dtype); +void SoftmaxOutputCompute(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + const SoftmaxOutputParam ¶m = nnvm::get(attrs.parsed); + const std::vector no_use_but_adapt_origin_api; + CHECK_EQ(inputs.size(), 2U); + + MSHADOW_REAL_TYPE_SWITCH(inputs[softmaxout_enum::kData].type_flag_, DType, { + SoftmaxOutputOp op(param); + op.Forward(ctx, inputs, req, outputs, no_use_but_adapt_origin_api); + }); +} + +template +void SoftmaxOutputGradCompute(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + const SoftmaxOutputParam& param = nnvm::get(attrs.parsed); + const std::vector no_use_but_adapt_origin_api; + CHECK_EQ(inputs.size(), 2U); + + std::vector out_grad{inputs[0]}; + std::vector out_data{inputs[0]}; + std::vector in_data(inputs.begin(), inputs.end()); + int dtype = inputs[0].type_flag_; + const std::vector &in_grad = outputs; + + MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { + SoftmaxOutputOp op(param); + op.Backward(ctx, out_grad, in_data, out_data, req, in_grad, no_use_but_adapt_origin_api); + }); +} + #if DMLC_USE_CXX11 class SoftmaxOutputProp : public OperatorProperty { @@ -414,4 +459,23 @@ class DeprecatedSoftmaxProp : public SoftmaxOutputProp { } // namespace op } // namespace mxnet + +namespace std { +template<> +struct hash { + size_t operator()(const mxnet::op::SoftmaxOutputParam& val) { + size_t ret = 0; + ret = dmlc::HashCombine(ret, val.grad_scale); + ret = dmlc::HashCombine(ret, val.ignore_label); + ret = dmlc::HashCombine(ret, val.multi_output); + ret = dmlc::HashCombine(ret, val.use_ignore); + ret = dmlc::HashCombine(ret, val.preserve_shape); + ret = dmlc::HashCombine(ret, val.normalization); + ret = dmlc::HashCombine(ret, val.out_grad); + ret = dmlc::HashCombine(ret, val.smooth_alpha); + return ret; + } +}; +} // namespace std + #endif // MXNET_OPERATOR_SOFTMAX_OUTPUT_INL_H_ diff --git a/src/operator/softmax_output.cc b/src/operator/softmax_output.cc index 5ba421fd195b..322ac0b93426 100644 --- a/src/operator/softmax_output.cc +++ b/src/operator/softmax_output.cc @@ -21,30 +21,137 @@ * Copyright (c) 2015 by Contributors * \file softmax_output.cc * \brief - * \author Bing Xu + * \author Bing Xu, Zhang Rong A */ #include "./softmax_output-inl.h" - +#if MXNET_USE_MKLDNN == 1 +#include "./nn/mkldnn/mkldnn_ops-inl.h" +#endif namespace mxnet { namespace op { -template<> -Operator *CreateOp(SoftmaxOutputParam param, int dtype) { - Operator *op = nullptr; - MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { - op = new SoftmaxOutputOp(param); - }) - return op; + +DMLC_REGISTER_PARAMETER(SoftmaxOutputParam); +struct SoftmaxOutputGrad { + const char *op_name; + std::vector operator()(const nnvm::NodePtr& n, + const std::vector& ograds) const { + std::vector out_data(n->num_outputs()); + for (uint32_t i = 0; i < out_data.size(); ++i) { + out_data[i] = nnvm::NodeEntry{n, i, 0}; + } + std::vector heads; + heads.push_back(out_data[softmaxout_enum::kOut]); + heads.push_back(n->inputs[softmaxout_enum::kLabel]); + + nnvm::NodePtr gnode = nnvm::Node::Create(); + gnode->inputs = std::move(heads); + gnode->control_deps.emplace_back(n); + gnode->attrs = n->attrs; + gnode->attrs.op = nnvm::Op::Get("_backward_SoftmaxOutput"); + gnode->attrs.name = n->attrs.name + "_backward"; + std::vector in_grad(2); + in_grad[0] = nnvm::NodeEntry{gnode, 0, 0}; + in_grad[1] = nnvm::NodeEntry{gnode, 1, 0}; + return in_grad; + } +}; + +static inline std::vector ListArguments() { + return {"data", "label"}; } -// DO_BIND_DISPATCH comes from operator_common.h -Operator *SoftmaxOutputProp::CreateOperatorEx(Context ctx, std::vector *in_shape, - std::vector *in_type) const { - DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); +static bool SoftmaxOutputType(const nnvm::NodeAttrs& attrs, + std::vector *in_type, + std::vector *out_type) { + CHECK_EQ(in_type->size(), 2U); + int dtype = (*in_type)[0]; + CHECK_NE(dtype, -1) << "First input must have specified type"; + for (size_t i = 0; i < in_type->size(); ++i) { + if ((*in_type)[i] == -1) { + (*in_type)[i] = dtype; + } else { + UNIFORM_TYPE_CHECK((*in_type)[i], dtype, ListArguments()[i]); + } + } + out_type->clear(); + out_type->push_back(dtype); + return true; } -DMLC_REGISTER_PARAMETER(SoftmaxOutputParam); +static bool SoftmaxOutputShape(const nnvm::NodeAttrs& attrs, + std::vector *in_shape, + std::vector *out_shape) { + using namespace mshadow; + const SoftmaxOutputParam& param = nnvm::get(attrs.parsed); + CHECK_EQ(in_shape->size(), 2U) << "Input:[data, label]"; + const TShape &dshape = in_shape->at(0); + if (dshape.ndim() == 0) return false; -MXNET_REGISTER_OP_PROPERTY(SoftmaxOutput, SoftmaxOutputProp) + // label.shape == data.shape: use probability as label + if (dshape != (*in_shape)[softmaxout_enum::kLabel]) { + if (param.multi_output) { + TShape lshape1 = Shape2(dshape[0], dshape.Size()/dshape[0]/dshape[1]); + TShape lshape2(dshape.ndim() - 1); + lshape2[0] = dshape[0]; + for (index_t i = 2; i < dshape.ndim(); ++i) + lshape2[i-1] = dshape[i]; + TShape lshape3 = dshape; + lshape3[1] = 1; + if (in_shape->at(softmaxout_enum::kLabel).ndim() == 0) { + in_shape->at(softmaxout_enum::kLabel) = lshape1; + } else if (in_shape->at(softmaxout_enum::kLabel) == lshape1) { + } else if (in_shape->at(softmaxout_enum::kLabel) == lshape2) { + } else if (in_shape->at(softmaxout_enum::kLabel) == lshape3) { + } else { + std::ostringstream os; + os << "Expecting " << lshape1 << " or " << lshape2 + << ". But got " << in_shape->at(softmaxout_enum::kLabel); + throw InferShapeError(os.str(), softmaxout_enum::kLabel); + } + } else { + TShape label_shape(dshape.ndim() - 1); + for (index_t i = 0; i + 1 < dshape.ndim(); ++i) + label_shape[i] = dshape[i]; + SHAPE_ASSIGN_CHECK(*in_shape, softmaxout_enum::kLabel, label_shape); + } + } + + out_shape->clear(); + out_shape->push_back(dshape); + return true; +} + +#if MXNET_USE_MKLDNN == 1 +inline static bool SoftmaxOutputStorageType(const nnvm::NodeAttrs& attrs, + const int dev_mask, + DispatchMode* dispatch_mode, + std::vector* in_attrs, + std::vector* out_attrs) { + CHECK_EQ(in_attrs->size(), 2); + CHECK_EQ(out_attrs->size(), 1); + + return MKLDNNStorageType(attrs, dev_mask, true, dispatch_mode, in_attrs, + out_attrs); +} + +void SoftmaxOutputComputeExCPU(const nnvm::NodeAttrs &attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs) { + CHECK_EQ(inputs.size(), 2U); + const SoftmaxOutputParam ¶m = nnvm::get(attrs.parsed); + if (SupportMKLDNN(inputs[0]) && !ctx.is_train && SupportMKLDNNSoftmaxOutput(param)) { + MKLDNN_OPCHECK_INIT(false, outputs.size(), inputs, outputs); + MKLDNNSoftmaxOutputForward(attrs, ctx, inputs, req, outputs); + MKLDNN_OPCHECK_RUN(SoftmaxOutputCompute, attrs, ctx, inputs, req, outputs); + return; + } + FallBackCompute(SoftmaxOutputCompute, attrs, ctx, inputs, req, outputs); +} +#endif + +NNVM_REGISTER_OP(SoftmaxOutput) .describe(R"code(Computes the gradient of cross entropy loss with respect to softmax output. - This operator computes the gradient in two steps. @@ -121,23 +228,41 @@ MXNET_REGISTER_OP_PROPERTY(SoftmaxOutput, SoftmaxOutputProp) - ``'valid'``: divide the gradient by the number of instances which are not ignored. )code" ADD_FILELINE) +.set_num_inputs(2) +.set_num_outputs(1) +.set_attr_parser(ParamParser) +#if MXNET_USE_MKLDNN == 1 +.set_attr("FInferStorageType", SoftmaxOutputStorageType) +.set_attr("TIsMKLDNN", true) +.set_attr("FComputeEx", SoftmaxOutputComputeExCPU) +#endif +.set_attr("FListInputNames", [](const NodeAttrs& attrs) { + return std::vector{"data", "label"}; +}) +.set_attr("FListOutputNames", [](const NodeAttrs& attrs) { + return std::vector{"output"}; +}) +.set_attr("FInferShape", SoftmaxOutputShape) +.set_attr("FInferType", SoftmaxOutputType) +.set_attr("FCompute", SoftmaxOutputCompute) +.set_attr("FGradient", SoftmaxOutputGrad{"_backward_SoftmaxOutput"}) +.set_attr("FInplaceOption", [](const NodeAttrs& attrs){ + return std::vector >{{0, 0}}; +}) .add_argument("data", "NDArray-or-Symbol", "Input array.") .add_argument("label", "NDArray-or-Symbol", "Ground truth label.") .add_arguments(SoftmaxOutputParam::__FIELDS__()); +// Softmax symbol is renamed to SoftmaxOutput and deprecated since Dec, 2015 +NNVM_REGISTER_OP(SoftmaxOutput).add_alias("Softmax"); -MXNET_REGISTER_OP_PROPERTY(Softmax, DeprecatedSoftmaxProp) -.describe(R"code(Please use `SoftmaxOutput`. - -.. note:: - - This operator has been renamed to `SoftmaxOutput`, which - computes the gradient of cross-entropy loss w.r.t softmax output. - To just compute softmax output, use the `softmax` operator. - -)code" ADD_FILELINE) -.add_argument("data", "NDArray-or-Symbol", "Input array.") -.add_arguments(SoftmaxOutputParam::__FIELDS__()); - +NNVM_REGISTER_OP(_backward_SoftmaxOutput) +.set_num_outputs(2) +.set_attr("TIsBackward", true) +.set_attr("FInplaceOption", [](const NodeAttrs& attrs){ + return std::vector >{{0, 0}}; +}) +.set_attr_parser(ParamParser) +.set_attr("FCompute", SoftmaxOutputGradCompute); } // namespace op } // namespace mxnet diff --git a/src/operator/softmax_output.cu b/src/operator/softmax_output.cu index afcc8f4fc6bd..b2a41672e92a 100644 --- a/src/operator/softmax_output.cu +++ b/src/operator/softmax_output.cu @@ -28,14 +28,12 @@ namespace mxnet { namespace op { -template<> -Operator *CreateOp(SoftmaxOutputParam param, int dtype) { - Operator *op = NULL; - MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { - op = new SoftmaxOutputOp(param); - }) - return op; -} + +NNVM_REGISTER_OP(SoftmaxOutput) +.set_attr("FCompute", SoftmaxOutputCompute); + +NNVM_REGISTER_OP(_backward_SoftmaxOutput) +.set_attr("FCompute", SoftmaxOutputGradCompute); } // namespace op } // namespace mxnet From c704f1f5831fa0a26fb98e8b2208c99797229c8d Mon Sep 17 00:00:00 2001 From: Lai Wei Date: Wed, 13 Feb 2019 09:54:22 -0800 Subject: [PATCH 10/75] follow up on fix nightly test (#14134) * fix nightly * trigger ci --- docs/tutorials/gluon/gluon_from_experiment_to_deployment.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md b/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md index 36c7a2e4dafa..a3a6aab3593c 100644 --- a/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md +++ b/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md @@ -52,7 +52,6 @@ Now let's first import necessary packages: import math import os import time -from multiprocessing import cpu_count from mxnet import autograd from mxnet import gluon, init @@ -77,7 +76,8 @@ lr_factor = 0.75 lr_epochs = [10, 20, 30] num_gpus = mx.context.num_gpus() -num_workers = cpu_count() +# you can replace num_workers with the number of cores on you device +num_workers = 8 ctx = [mx.gpu(i) for i in range(num_gpus)] if num_gpus > 0 else [mx.cpu()] batch_size = per_device_batch_size * max(num_gpus, 1) ``` From 199bc7eef600484ed8c602cc73e2fb2c183aedcd Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Wed, 13 Feb 2019 10:12:12 -0800 Subject: [PATCH 11/75] make rat-excludes compliant with apache release policy (#14142) * remove md from whitelist * remove rst from whitelist * remove docker folder from whitelist * remove docs from whitelist --- CONTRIBUTORS.md | 19 ++++++++++ MKLDNN_README.md | 19 ++++++++++ NEWS.md | 19 ++++++++++ README.md | 19 ++++++++++ amalgamation/README.md | 19 ++++++++++ ci/README.md | 19 ++++++++++ ci/docker/install/docs_requirements | 20 ++++++++++ ci/docker/qemu/README.md | 19 ++++++++++ ci/publish/README.md | 19 ++++++++++ ci/qemu/README.md | 19 ++++++++++ contrib/clojure-package/README.md | 19 ++++++++++ .../examples/captcha/README.md | 19 ++++++++++ .../cnn-text-classification/README.md | 19 ++++++++++ .../clojure-package/examples/gan/README.md | 19 ++++++++++ .../examples/imclassification/README.md | 19 ++++++++++ .../examples/infer/imageclassifier/README.md | 19 ++++++++++ .../examples/infer/objectdetector/README.md | 19 ++++++++++ .../examples/infer/predictor/README.md | 19 ++++++++++ .../clojure-package/examples/module/README.md | 19 ++++++++++ .../examples/multi-label/README.md | 19 ++++++++++ .../examples/neural-style/README.md | 19 ++++++++++ .../examples/pre-trained-models/README.md | 19 ++++++++++ .../examples/profiler/README.md | 19 ++++++++++ .../clojure-package/examples/rnn/README.md | 19 ++++++++++ .../examples/tutorial/README.md | 19 ++++++++++ .../examples/visualization/README.md | 19 ++++++++++ contrib/clojure-package/testing.md | 19 ++++++++++ cpp-package/README.md | 19 ++++++++++ cpp-package/example/README.md | 19 ++++++++++ cpp-package/example/feature_extract/README.md | 19 ++++++++++ cpp-package/example/inference/README.md | 19 ++++++++++ docker/Dockerfiles/Dockerfile.in.julia | 19 ++++++++++ docker/Dockerfiles/Dockerfile.in.lib.cpu | 19 ++++++++++ docker/Dockerfiles/Dockerfile.in.lib.gpu | 19 ++++++++++ docker/Dockerfiles/Dockerfile.in.perl | 19 ++++++++++ docker/Dockerfiles/Dockerfile.in.python | 19 ++++++++++ docker/Dockerfiles/Dockerfile.in.r-lang | 19 ++++++++++ docker/Dockerfiles/Dockerfile.in.scala | 19 ++++++++++ docker/Dockerfiles/Dockerfile.tensorrt | 2 + docker/Dockerfiles/License.md | 20 ---------- docker/README.md | 19 ++++++++++ docker/docker-python/README.md | 37 ++++++++++++++----- docs/Dockerfile | 20 ++++++++++ docs/README.md | 19 ++++++++++ docs/api/c++/index.md | 19 ++++++++++ docs/api/clojure/index.md | 19 ++++++++++ docs/api/clojure/kvstore.md | 19 ++++++++++ docs/api/clojure/module.md | 19 ++++++++++ docs/api/clojure/ndarray.md | 19 ++++++++++ docs/api/clojure/symbol.md | 19 ++++++++++ docs/api/clojure/symbol_in_pictures.md | 19 ++++++++++ docs/api/index.md | 19 ++++++++++ docs/api/java/index.md | 19 ++++++++++ docs/api/julia/index.md | 19 ++++++++++ docs/api/perl/index.md | 19 ++++++++++ docs/api/perl/io.md | 19 ++++++++++ docs/api/perl/kvstore.md | 19 ++++++++++ docs/api/perl/module.md | 19 ++++++++++ docs/api/perl/ndarray.md | 19 ++++++++++ docs/api/perl/symbol.md | 19 ++++++++++ docs/api/python/autograd/autograd.md | 19 ++++++++++ docs/api/python/callback/callback.md | 19 ++++++++++ docs/api/python/contrib/contrib.md | 19 ++++++++++ docs/api/python/contrib/onnx.md | 19 ++++++++++ docs/api/python/contrib/svrg_optimization.md | 19 ++++++++++ docs/api/python/contrib/text.md | 19 ++++++++++ docs/api/python/executor/executor.md | 19 ++++++++++ docs/api/python/gluon/contrib.md | 19 ++++++++++ docs/api/python/gluon/data.md | 19 ++++++++++ docs/api/python/gluon/gluon.md | 19 ++++++++++ docs/api/python/gluon/loss.md | 19 ++++++++++ docs/api/python/gluon/model_zoo.md | 19 ++++++++++ docs/api/python/gluon/nn.md | 19 ++++++++++ docs/api/python/gluon/rnn.md | 19 ++++++++++ docs/api/python/image/image.md | 19 ++++++++++ docs/api/python/index.md | 19 ++++++++++ docs/api/python/io/io.md | 19 ++++++++++ docs/api/python/kvstore/kvstore.md | 19 ++++++++++ docs/api/python/metric/metric.md | 19 ++++++++++ docs/api/python/model.md | 19 ++++++++++ docs/api/python/module/module.md | 19 ++++++++++ docs/api/python/ndarray/contrib.md | 19 ++++++++++ docs/api/python/ndarray/linalg.md | 19 ++++++++++ docs/api/python/ndarray/ndarray.md | 19 ++++++++++ docs/api/python/ndarray/random.md | 19 ++++++++++ docs/api/python/ndarray/sparse.md | 19 ++++++++++ docs/api/python/optimization/contrib.md | 19 ++++++++++ docs/api/python/optimization/optimization.md | 19 ++++++++++ docs/api/python/rtc/rtc.md | 19 ++++++++++ docs/api/python/symbol/contrib.md | 19 ++++++++++ docs/api/python/symbol/linalg.md | 19 ++++++++++ docs/api/python/symbol/random.md | 19 ++++++++++ docs/api/python/symbol/rnn.md | 19 ++++++++++ docs/api/python/symbol/sparse.md | 19 ++++++++++ docs/api/python/symbol/symbol.md | 19 ++++++++++ .../symbol_in_pictures/symbol_in_pictures.md | 19 ++++++++++ docs/api/python/tools/test_utils.md | 19 ++++++++++ docs/api/python/tools/visualization.md | 19 ++++++++++ docs/api/r/index.md | 19 ++++++++++ docs/api/scala/index.md | 19 ++++++++++ docs/api/scala/infer.md | 19 ++++++++++ docs/api/scala/io.md | 19 ++++++++++ docs/api/scala/kvstore.md | 19 ++++++++++ docs/api/scala/model.md | 19 ++++++++++ docs/api/scala/module.md | 19 ++++++++++ docs/api/scala/ndarray.md | 19 ++++++++++ docs/api/scala/symbol.md | 19 ++++++++++ docs/api/scala/symbol_in_pictures.md | 19 ++++++++++ docs/architecture/exception_handling.md | 19 ++++++++++ docs/architecture/index.md | 19 ++++++++++ docs/architecture/note_data_loading.md | 19 ++++++++++ docs/architecture/note_engine.md | 19 ++++++++++ docs/architecture/note_memory.md | 19 ++++++++++ docs/architecture/overview.md | 19 ++++++++++ docs/architecture/program_model.md | 19 ++++++++++ docs/architecture/rnn_interface.md | 19 ++++++++++ docs/build_version_doc/Dockerfile | 20 ++++++++++ docs/build_version_doc/README.md | 19 ++++++++++ docs/community/contribute.md | 19 ++++++++++ docs/community/ecosystem.md | 19 ++++++++++ docs/community/index.md | 19 ++++++++++ docs/community/mxnet_channels.md | 19 ++++++++++ docs/community/powered_by.md | 19 ++++++++++ docs/faq/add_op_in_backend.md | 19 ++++++++++ docs/faq/bucketing.md | 19 ++++++++++ docs/faq/caffe.md | 19 ++++++++++ docs/faq/cloud.md | 19 ++++++++++ docs/faq/develop_and_hack.md | 19 ++++++++++ docs/faq/distributed_training.md | 19 ++++++++++ docs/faq/env_var.md | 19 ++++++++++ docs/faq/faq.md | 19 ++++++++++ docs/faq/finetune.md | 19 ++++++++++ docs/faq/float16.md | 19 ++++++++++ docs/faq/gradient_compression.md | 19 ++++++++++ docs/faq/index.md | 19 ++++++++++ docs/faq/model_parallel_lstm.md | 19 ++++++++++ docs/faq/multi_devices.md | 19 ++++++++++ docs/faq/new_op.md | 19 ++++++++++ docs/faq/nnpack.md | 19 ++++++++++ docs/faq/perf.md | 19 ++++++++++ docs/faq/recordio.md | 19 ++++++++++ docs/faq/s3_integration.md | 19 ++++++++++ docs/faq/security.md | 19 ++++++++++ docs/faq/smart_device.md | 19 ++++++++++ docs/faq/visualize_graph.md | 19 ++++++++++ docs/faq/why_mxnet.md | 19 ++++++++++ docs/gluon/index.md | 19 ++++++++++ docs/index.md | 19 ++++++++++ docs/install/amazonlinux_setup.md | 19 ++++++++++ docs/install/build_from_source.md | 19 ++++++++++ docs/install/c_plus_plus.md | 19 ++++++++++ docs/install/centos_setup.md | 19 ++++++++++ docs/install/download.md | 19 ++++++++++ docs/install/index.md | 19 ++++++++++ docs/install/java_setup.md | 19 ++++++++++ docs/install/osx_setup.md | 19 ++++++++++ docs/install/raspbian_setup.md | 19 ++++++++++ docs/install/scala_setup.md | 19 ++++++++++ docs/install/tx2_setup.md | 19 ++++++++++ docs/install/ubuntu_setup.md | 19 ++++++++++ docs/install/validate_mxnet.md | 19 ++++++++++ docs/install/windows_setup.md | 19 ++++++++++ docs/model_zoo/index.md | 19 ++++++++++ docs/settings.ini | 17 +++++++++ docs/tutorials/basic/data.md | 19 ++++++++++ docs/tutorials/basic/index.md | 19 ++++++++++ docs/tutorials/basic/module.md | 19 ++++++++++ docs/tutorials/basic/ndarray.md | 19 ++++++++++ docs/tutorials/basic/ndarray_indexing.md | 19 ++++++++++ docs/tutorials/basic/reshape_transpose.md | 19 ++++++++++ docs/tutorials/basic/symbol.md | 19 ++++++++++ docs/tutorials/c++/basics.md | 19 ++++++++++ docs/tutorials/c++/index.md | 19 ++++++++++ .../c++/mxnet_cpp_inference_tutorial.md | 19 ++++++++++ docs/tutorials/c++/subgraphAPI.md | 19 ++++++++++ .../control_flow/ControlFlowTutorial.md | 19 ++++++++++ docs/tutorials/control_flow/index.md | 19 ++++++++++ docs/tutorials/embedded/index.md | 19 ++++++++++ docs/tutorials/embedded/wine_detector.md | 19 ++++++++++ docs/tutorials/gluon/autograd.md | 19 ++++++++++ docs/tutorials/gluon/custom_layer.md | 19 ++++++++++ docs/tutorials/gluon/customop.md | 19 ++++++++++ docs/tutorials/gluon/data_augmentation.md | 19 ++++++++++ docs/tutorials/gluon/datasets.md | 19 ++++++++++ docs/tutorials/gluon/gluon.md | 19 ++++++++++ .../gluon_from_experiment_to_deployment.md | 19 ++++++++++ .../tutorials/gluon/gotchas_numpy_in_mxnet.md | 19 ++++++++++ docs/tutorials/gluon/hybrid.md | 19 ++++++++++ docs/tutorials/gluon/index.md | 19 ++++++++++ docs/tutorials/gluon/info_gan.md | 19 ++++++++++ docs/tutorials/gluon/learning_rate_finder.md | 19 ++++++++++ .../gluon/learning_rate_schedules.md | 19 ++++++++++ .../gluon/learning_rate_schedules_advanced.md | 19 ++++++++++ .../gluon/logistic_regression_explained.md | 19 ++++++++++ docs/tutorials/gluon/mnist.md | 19 ++++++++++ docs/tutorials/gluon/naming.md | 19 ++++++++++ docs/tutorials/gluon/ndarray.md | 19 ++++++++++ docs/tutorials/gluon/pretrained_models.md | 19 ++++++++++ docs/tutorials/gluon/save_load_params.md | 19 ++++++++++ docs/tutorials/index.md | 19 ++++++++++ docs/tutorials/java/index.md | 19 ++++++++++ docs/tutorials/java/mxnet_java_on_intellij.md | 19 ++++++++++ docs/tutorials/java/ssd_inference.md | 19 ++++++++++ docs/tutorials/nlp/cnn.md | 19 ++++++++++ docs/tutorials/nlp/index.md | 19 ++++++++++ docs/tutorials/onnx/export_mxnet_to_onnx.md | 19 ++++++++++ docs/tutorials/onnx/fine_tuning_gluon.md | 19 ++++++++++ docs/tutorials/onnx/index.md | 19 ++++++++++ .../tutorials/onnx/inference_on_onnx_model.md | 19 ++++++++++ docs/tutorials/onnx/super_resolution.md | 19 ++++++++++ docs/tutorials/python/data_augmentation.md | 19 ++++++++++ .../python/data_augmentation_with_masks.md | 19 ++++++++++ docs/tutorials/python/index.md | 19 ++++++++++ docs/tutorials/python/kvstore.md | 19 ++++++++++ docs/tutorials/python/linear-regression.md | 19 ++++++++++ docs/tutorials/python/matrix_factorization.md | 19 ++++++++++ docs/tutorials/python/mnist.md | 19 ++++++++++ docs/tutorials/python/predict_image.md | 19 ++++++++++ docs/tutorials/python/profiler.md | 19 ++++++++++ .../python/types_of_data_augmentation.md | 19 ++++++++++ docs/tutorials/r/CallbackFunction.md | 19 ++++++++++ docs/tutorials/r/CustomIterator.md | 19 ++++++++++ docs/tutorials/r/CustomLossFunction.md | 19 ++++++++++ docs/tutorials/r/MultidimLstm.md | 19 ++++++++++ docs/tutorials/r/charRnnModel.md | 19 ++++++++++ .../r/classifyRealImageWithPretrainedModel.md | 19 ++++++++++ docs/tutorials/r/fiveMinutesNeuralNetwork.md | 19 ++++++++++ docs/tutorials/r/index.md | 19 ++++++++++ docs/tutorials/r/mnistCompetition.md | 19 ++++++++++ docs/tutorials/r/ndarray.md | 19 ++++++++++ docs/tutorials/r/symbol.md | 19 ++++++++++ docs/tutorials/scala/char_lstm.md | 19 ++++++++++ docs/tutorials/scala/index.md | 19 ++++++++++ docs/tutorials/scala/mnist.md | 19 ++++++++++ .../scala/mxnet_scala_on_intellij.md | 19 ++++++++++ docs/tutorials/sparse/csr.md | 19 ++++++++++ docs/tutorials/sparse/index.md | 19 ++++++++++ docs/tutorials/sparse/row_sparse.md | 19 ++++++++++ docs/tutorials/sparse/train.md | 19 ++++++++++ docs/tutorials/speech_recognition/ctc.md | 19 ++++++++++ docs/tutorials/speech_recognition/index.md | 19 ++++++++++ docs/tutorials/tensorrt/index.md | 19 ++++++++++ docs/tutorials/tensorrt/inference_with_trt.md | 19 ++++++++++ docs/tutorials/unsupervised_learning/gan.md | 19 ++++++++++ docs/tutorials/unsupervised_learning/index.md | 19 ++++++++++ docs/tutorials/vision/cnn_visualization.md | 19 ++++++++++ docs/tutorials/vision/index.md | 19 ++++++++++ .../vision/large_scale_classification.md | 19 ++++++++++ example/README.md | 19 ++++++++++ example/adversary/README.md | 19 ++++++++++ example/autoencoder/README.md | 19 ++++++++++ .../variational_autoencoder/README.md | 19 ++++++++++ example/bayesian-methods/README.md | 19 ++++++++++ example/bi-lstm-sort/README.md | 19 ++++++++++ example/caffe/README.md | 19 ++++++++++ example/capsnet/README.md | 19 ++++++++++ example/captcha/README.md | 19 ++++++++++ .../cnn_chinese_text_classification/README.md | 19 ++++++++++ example/cnn_text_classification/README.md | 19 ++++++++++ example/ctc/README.md | 19 ++++++++++ example/deep-embedded-clustering/README.md | 19 ++++++++++ example/distributed_training/README.md | 19 ++++++++++ example/dsd/README.md | 19 ++++++++++ example/fcn-xs/README.md | 19 ++++++++++ example/gluon/audio/urban_sounds/README.md | 19 ++++++++++ example/gluon/dc_gan/README.md | 19 ++++++++++ example/gluon/embedding_learning/README.md | 19 ++++++++++ example/gluon/sn_gan/README.md | 19 ++++++++++ example/gluon/style_transfer/README.md | 19 ++++++++++ example/gluon/tree_lstm/README.md | 19 ++++++++++ example/gluon/word_language_model/README.md | 19 ++++++++++ example/kaggle-ndsb1/README.md | 19 ++++++++++ example/kaggle-ndsb2/README.md | 19 ++++++++++ .../matrix_factorization/README.md | 19 ++++++++++ example/module/README.md | 19 ++++++++++ example/multi-task/README.md | 19 ++++++++++ example/multivariate_time_series/README.md | 19 ++++++++++ example/named_entity_recognition/README.md | 19 ++++++++++ example/nce-loss/README.md | 19 ++++++++++ example/neural-style/README.md | 19 ++++++++++ example/neural-style/end_to_end/README.md | 19 ++++++++++ example/numpy-ops/README.md | 19 ++++++++++ example/profiler/README.md | 19 ++++++++++ example/rcnn/README.md | 19 ++++++++++ example/recommenders/README.md | 19 ++++++++++ example/reinforcement-learning/a3c/README.md | 19 ++++++++++ example/reinforcement-learning/ddpg/README.md | 19 ++++++++++ .../parallel_actor_critic/README.md | 19 ++++++++++ .../restricted-boltzmann-machine/README.md | 19 ++++++++++ example/rnn/README.md | 19 ++++++++++ example/rnn/bucketing/README.md | 19 ++++++++++ example/rnn/old/README.md | 19 ++++++++++ example/rnn/word_lm/README.md | 19 ++++++++++ .../sparse/factorization_machine/README.md | 19 ++++++++++ .../sparse/linear_classification/README.md | 19 ++++++++++ example/sparse/matrix_factorization/README.md | 19 ++++++++++ example/sparse/wide_deep/README.md | 19 ++++++++++ example/speech_recognition/README.md | 19 ++++++++++ example/ssd/README.md | 19 ++++++++++ example/ssd/dataset/pycocotools/README.md | 19 ++++++++++ example/ssd/model/README.md | 19 ++++++++++ example/ssd/symbol/README.md | 19 ++++++++++ example/ssd/tools/caffe_converter/README.md | 19 ++++++++++ example/stochastic-depth/README.md | 19 ++++++++++ example/svm_mnist/README.md | 19 ++++++++++ example/svrg_module/README.md | 19 ++++++++++ example/vae-gan/README.md | 19 ++++++++++ julia/LICENSE.md | 19 ++++++++++ julia/NEWS.md | 19 ++++++++++ julia/README-DEV.md | 19 ++++++++++ julia/README.md | 19 ++++++++++ julia/docs/src/api.md | 19 ++++++++++ julia/docs/src/api/callback.md | 19 ++++++++++ julia/docs/src/api/context.md | 19 ++++++++++ julia/docs/src/api/executor.md | 19 ++++++++++ julia/docs/src/api/initializer.md | 19 ++++++++++ julia/docs/src/api/io.md | 19 ++++++++++ julia/docs/src/api/kvstore.md | 19 ++++++++++ julia/docs/src/api/metric.md | 19 ++++++++++ julia/docs/src/api/model.md | 19 ++++++++++ julia/docs/src/api/ndarray.md | 19 ++++++++++ julia/docs/src/api/nn-factory.md | 19 ++++++++++ julia/docs/src/api/symbolic-node.md | 19 ++++++++++ julia/docs/src/api/visualize.md | 19 ++++++++++ julia/docs/src/index.md | 19 ++++++++++ julia/docs/src/tutorial/char-lstm.md | 19 ++++++++++ julia/docs/src/tutorial/mnist.md | 19 ++++++++++ julia/docs/src/user-guide/faq.md | 19 ++++++++++ julia/docs/src/user-guide/install.md | 19 ++++++++++ julia/docs/src/user-guide/overview.md | 19 ++++++++++ julia/examples/char-lstm/README.md | 19 ++++++++++ julia/plugins/README.md | 19 ++++++++++ matlab/README.md | 19 ++++++++++ .../examples/gluon/style_transfer/README.md | 19 ++++++++++ .../sparse/matrix_factorization/README.md | 19 ++++++++++ .../examples/sparse/wide_deep/README.md | 19 ++++++++++ perl-package/README.md | 19 ++++++++++ plugin/caffe/README.md | 19 ++++++++++ python/README.md | 19 ++++++++++ python/minpy/README.md | 4 -- scala-package/README.md | 19 ++++++++++ .../javaapi/infer/objectdetector/README.md | 19 ++++++++++ .../javaapi/infer/predictor/README.md | 19 ++++++++++ .../apache/mxnetexamples/benchmark/README.md | 19 ++++++++++ .../cnntextclassification/README.md | 19 ++++++++++ .../apache/mxnetexamples/customop/README.md | 19 ++++++++++ .../org/apache/mxnetexamples/gan/README.md | 19 ++++++++++ .../mxnetexamples/imclassification/README.md | 19 ++++++++++ .../infer/imageclassifier/README.md | 19 ++++++++++ .../infer/objectdetector/README.md | 19 ++++++++++ .../mxnetexamples/neuralstyle/README.md | 19 ++++++++++ .../org/apache/mxnetexamples/rnn/README.md | 19 ++++++++++ scala-package/memory-management.md | 19 ++++++++++ scala-package/mxnet-demo/java-demo/README.md | 19 ++++++++++ scala-package/mxnet-demo/scala-demo/README.md | 19 ++++++++++ scala-package/native/README.md | 19 ++++++++++ scala-package/packageTest/README.md | 19 ++++++++++ scala-package/spark/README.md | 19 ++++++++++ tests/README.md | 19 ++++++++++ tests/nightly/README.md | 19 ++++++++++ .../apache_rat_license_check/README.md | 19 ++++++++++ .../apache_rat_license_check/rat-excludes | 6 +-- .../broken_link_checker_test/README.md | 19 ++++++++++ .../README.md | 19 ++++++++++ tests/nightly/straight_dope/README.md | 19 ++++++++++ tests/python/README.md | 19 ++++++++++ tools/accnn/README.md | 19 ++++++++++ tools/bandwidth/README.md | 19 ++++++++++ tools/caffe_converter/README.md | 19 ++++++++++ tools/caffe_translator/README.md | 19 ++++++++++ tools/caffe_translator/build_from_source.md | 19 ++++++++++ tools/caffe_translator/faq.md | 19 ++++++++++ tools/cfn/Readme.md | 19 ++++++++++ tools/coreml/README.md | 19 ++++++++++ tools/coreml/pip_package/README.rst | 17 +++++++++ tools/dependencies/README.md | 19 ++++++++++ tools/staticbuild/README.md | 19 ++++++++++ 377 files changed, 7099 insertions(+), 37 deletions(-) delete mode 100644 docker/Dockerfiles/License.md mode change 100755 => 100644 docs/faq/why_mxnet.md mode change 100755 => 100644 docs/install/windows_setup.md mode change 100755 => 100644 example/ssd/dataset/pycocotools/README.md delete mode 100644 python/minpy/README.md mode change 100755 => 100644 tests/nightly/README.md mode change 100755 => 100644 tests/nightly/apache_rat_license_check/README.md mode change 100755 => 100644 tests/nightly/broken_link_checker_test/README.md mode change 100755 => 100644 tests/nightly/straight_dope/README.md diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 6bf44c55de6d..96736b76f136 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1,3 +1,22 @@ + + Contributors of Apache MXNet (incubating) ========================================= MXNet has been developed by a community of people who are interested in large-scale machine learning and deep learning. diff --git a/MKLDNN_README.md b/MKLDNN_README.md index ecb721e5fffe..7068b8519b39 100644 --- a/MKLDNN_README.md +++ b/MKLDNN_README.md @@ -1,3 +1,22 @@ + + # Build/Install MXNet with MKL-DNN A better training and inference performance is expected to be achieved on Intel-Architecture CPUs with MXNet built with [Intel MKL-DNN](https://github.com/intel/mkl-dnn) on multiple operating system, including Linux, Windows and MacOS. diff --git a/NEWS.md b/NEWS.md index f06cc35d8b0f..95fa8a52b047 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,22 @@ + + MXNet Change Log ================ diff --git a/README.md b/README.md index f5d40a6f656b..6955ddd3ed3d 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,22 @@ + +

diff --git a/amalgamation/README.md b/amalgamation/README.md index 3faf4d1e18a4..aaf01b6b6e7f 100644 --- a/amalgamation/README.md +++ b/amalgamation/README.md @@ -1,3 +1,22 @@ + + MXNet Amalgamation ================== This folder contains a amalgamation generation script to generate the entire mxnet library into one file. diff --git a/ci/README.md b/ci/README.md index f56a6f6a7978..6ad8cffa1550 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,3 +1,22 @@ + + # Containerized build & test utilities This folder contains scripts and dockerfiles used to build and test MXNet using diff --git a/ci/docker/install/docs_requirements b/ci/docker/install/docs_requirements index 4e3ce3e55e0b..3cfef1e33901 100644 --- a/ci/docker/install/docs_requirements +++ b/ci/docker/install/docs_requirements @@ -1,3 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + beautifulsoup4==4.6.3 breathe==4.10.0 cpplint==1.3.0 diff --git a/ci/docker/qemu/README.md b/ci/docker/qemu/README.md index 1dcaa5aeb607..bf63ed9fc89c 100644 --- a/ci/docker/qemu/README.md +++ b/ci/docker/qemu/README.md @@ -1 +1,20 @@ + + These are files used in the docker container that runs QEMU diff --git a/ci/publish/README.md b/ci/publish/README.md index f1ece6f84097..2beb2b399f8b 100644 --- a/ci/publish/README.md +++ b/ci/publish/README.md @@ -1,3 +1,22 @@ + + # MXNet Publish Settings This folder contains the configuration for restricted nodes on Jenkins for the publishing MXNet artifacts. It also contains a folder called `scala` that contains everything required for publishing to Maven. In this `README`, we provide a brief walkthrough of the Jenkins configuration as well as the usage of the Scala deployment files. Python publishing is TBD. diff --git a/ci/qemu/README.md b/ci/qemu/README.md index 6dde8916b28c..4b5e0db3047f 100644 --- a/ci/qemu/README.md +++ b/ci/qemu/README.md @@ -1,3 +1,22 @@ + + # QEMU base image creation This folder contains scripts and configuration to create a QEMU virtual drive with a debian system. diff --git a/contrib/clojure-package/README.md b/contrib/clojure-package/README.md index 840c15e75033..6c20fe0b609c 100644 --- a/contrib/clojure-package/README.md +++ b/contrib/clojure-package/README.md @@ -1,3 +1,22 @@ + + # Clojure MXNet A Clojure Package Built on the MXNet Deep Learning Library diff --git a/contrib/clojure-package/examples/captcha/README.md b/contrib/clojure-package/examples/captcha/README.md index 6b593b2f1c65..51a407b6e04c 100644 --- a/contrib/clojure-package/examples/captcha/README.md +++ b/contrib/clojure-package/examples/captcha/README.md @@ -1,3 +1,22 @@ + + # Captcha This is the clojure version of [captcha recognition](https://github.com/xlvector/learning-dl/tree/master/mxnet/ocr) diff --git a/contrib/clojure-package/examples/cnn-text-classification/README.md b/contrib/clojure-package/examples/cnn-text-classification/README.md index 19bb9137334b..b1ae538e5281 100644 --- a/contrib/clojure-package/examples/cnn-text-classification/README.md +++ b/contrib/clojure-package/examples/cnn-text-classification/README.md @@ -1,3 +1,22 @@ + + # cnn-text-classification An example of text classification using CNN diff --git a/contrib/clojure-package/examples/gan/README.md b/contrib/clojure-package/examples/gan/README.md index 2b46a6cf3e83..8c2c195f0ccc 100644 --- a/contrib/clojure-package/examples/gan/README.md +++ b/contrib/clojure-package/examples/gan/README.md @@ -1,3 +1,22 @@ + + # gan This is an example of how to do a GAN with the MNIST data diff --git a/contrib/clojure-package/examples/imclassification/README.md b/contrib/clojure-package/examples/imclassification/README.md index 4677f289d7e0..f9aa30f1b0c9 100644 --- a/contrib/clojure-package/examples/imclassification/README.md +++ b/contrib/clojure-package/examples/imclassification/README.md @@ -1,3 +1,22 @@ + + # imclassification This shows off how to do image classification with the module api diff --git a/contrib/clojure-package/examples/infer/imageclassifier/README.md b/contrib/clojure-package/examples/infer/imageclassifier/README.md index a8328607c9a2..ec772d2c3980 100644 --- a/contrib/clojure-package/examples/infer/imageclassifier/README.md +++ b/contrib/clojure-package/examples/infer/imageclassifier/README.md @@ -1,3 +1,22 @@ + + # imageclassifier Run image classification using clojure infer package. diff --git a/contrib/clojure-package/examples/infer/objectdetector/README.md b/contrib/clojure-package/examples/infer/objectdetector/README.md index ec092a296a96..b9e216eb17ad 100644 --- a/contrib/clojure-package/examples/infer/objectdetector/README.md +++ b/contrib/clojure-package/examples/infer/objectdetector/README.md @@ -1,3 +1,22 @@ + + # objectdetector Run object detection on images using clojure infer package. diff --git a/contrib/clojure-package/examples/infer/predictor/README.md b/contrib/clojure-package/examples/infer/predictor/README.md index 9ca71cf469a0..9b9be7f7a35a 100644 --- a/contrib/clojure-package/examples/infer/predictor/README.md +++ b/contrib/clojure-package/examples/infer/predictor/README.md @@ -1,3 +1,22 @@ + + # predictor Run model prediction using clojure infer package. diff --git a/contrib/clojure-package/examples/module/README.md b/contrib/clojure-package/examples/module/README.md index 1b08a52f7db3..421720fe3896 100644 --- a/contrib/clojure-package/examples/module/README.md +++ b/contrib/clojure-package/examples/module/README.md @@ -1,3 +1,22 @@ + + ## Instructions This shows off how to use the module api. diff --git a/contrib/clojure-package/examples/multi-label/README.md b/contrib/clojure-package/examples/multi-label/README.md index 27a8c1ff01ff..bc2a0cca3e88 100644 --- a/contrib/clojure-package/examples/multi-label/README.md +++ b/contrib/clojure-package/examples/multi-label/README.md @@ -1,3 +1,22 @@ + + # multi-label This is a quick example of doing multi-label classification. diff --git a/contrib/clojure-package/examples/neural-style/README.md b/contrib/clojure-package/examples/neural-style/README.md index e05c31552dcd..fa27e49eab7a 100644 --- a/contrib/clojure-package/examples/neural-style/README.md +++ b/contrib/clojure-package/examples/neural-style/README.md @@ -1,3 +1,22 @@ + + # neural-style An example of neural style transfer diff --git a/contrib/clojure-package/examples/pre-trained-models/README.md b/contrib/clojure-package/examples/pre-trained-models/README.md index b0996da69a10..b3df9af8ef8d 100644 --- a/contrib/clojure-package/examples/pre-trained-models/README.md +++ b/contrib/clojure-package/examples/pre-trained-models/README.md @@ -1,3 +1,22 @@ + + # pre-trained-models This shows examples of how to use the pretrained models. MXNet comes with a number of pretrained models diff --git a/contrib/clojure-package/examples/profiler/README.md b/contrib/clojure-package/examples/profiler/README.md index d8a98d35dbe3..73482c09d6ee 100644 --- a/contrib/clojure-package/examples/profiler/README.md +++ b/contrib/clojure-package/examples/profiler/README.md @@ -1,3 +1,22 @@ + + # profiler An example of using the profiler. diff --git a/contrib/clojure-package/examples/rnn/README.md b/contrib/clojure-package/examples/rnn/README.md index cad3909447b1..379e1e40c65a 100644 --- a/contrib/clojure-package/examples/rnn/README.md +++ b/contrib/clojure-package/examples/rnn/README.md @@ -1,3 +1,22 @@ + + # rnn diff --git a/contrib/clojure-package/examples/tutorial/README.md b/contrib/clojure-package/examples/tutorial/README.md index cd19eb232760..625002e06bdd 100644 --- a/contrib/clojure-package/examples/tutorial/README.md +++ b/contrib/clojure-package/examples/tutorial/README.md @@ -1,3 +1,22 @@ + + # tutorial Tutorials are based on the Scala api examples here https://mxnet.incubator.apache.org/api/scala/ndarray.html diff --git a/contrib/clojure-package/examples/visualization/README.md b/contrib/clojure-package/examples/visualization/README.md index 8c6e2c2b3b79..6e951542ddca 100644 --- a/contrib/clojure-package/examples/visualization/README.md +++ b/contrib/clojure-package/examples/visualization/README.md @@ -1,3 +1,22 @@ + + # visualization Run `lein run` to have a sample network visualization printed for you diff --git a/contrib/clojure-package/testing.md b/contrib/clojure-package/testing.md index 8f87c2a76e81..66de164da540 100644 --- a/contrib/clojure-package/testing.md +++ b/contrib/clojure-package/testing.md @@ -1,3 +1,22 @@ + + ## Help with Testing If you want to give the repo a spin and help make it stable and ready for prime time that would be awesome. diff --git a/cpp-package/README.md b/cpp-package/README.md index 45941555b21c..524fbadb87ee 100644 --- a/cpp-package/README.md +++ b/cpp-package/README.md @@ -1,3 +1,22 @@ + + # MXNet C++ Package The MXNet C++ Package provides C++ API bindings to the users of MXNet. Currently, these bindings are not available as standalone package. diff --git a/cpp-package/example/README.md b/cpp-package/example/README.md index 724478f3d68f..00c6e96f6f27 100644 --- a/cpp-package/example/README.md +++ b/cpp-package/example/README.md @@ -1,3 +1,22 @@ + + # MXNet C++ Package Examples ## Building C++ examples diff --git a/cpp-package/example/feature_extract/README.md b/cpp-package/example/feature_extract/README.md index 87cfcb4021f0..7241bcd6425a 100644 --- a/cpp-package/example/feature_extract/README.md +++ b/cpp-package/example/feature_extract/README.md @@ -1,3 +1,22 @@ + + This example shows how to extract features with a pretrained model. Execute `run.sh` to: diff --git a/cpp-package/example/inference/README.md b/cpp-package/example/inference/README.md index efd2357570c8..322c92c75208 100644 --- a/cpp-package/example/inference/README.md +++ b/cpp-package/example/inference/README.md @@ -1,3 +1,22 @@ + + # MXNet C++ Package Inference Workflow Examples ## Building C++ Inference examples diff --git a/docker/Dockerfiles/Dockerfile.in.julia b/docker/Dockerfiles/Dockerfile.in.julia index 42422ddbed54..794f958c9c09 100644 --- a/docker/Dockerfiles/Dockerfile.in.julia +++ b/docker/Dockerfiles/Dockerfile.in.julia @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # part of the dockerfile to install the julia binding COPY install/julia.sh install/ diff --git a/docker/Dockerfiles/Dockerfile.in.lib.cpu b/docker/Dockerfiles/Dockerfile.in.lib.cpu index 002e2d1e4209..c6de40c5cea3 100644 --- a/docker/Dockerfiles/Dockerfile.in.lib.cpu +++ b/docker/Dockerfiles/Dockerfile.in.lib.cpu @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # dockerfile to build libmxnet.so on CPU FROM ubuntu:14.04 diff --git a/docker/Dockerfiles/Dockerfile.in.lib.gpu b/docker/Dockerfiles/Dockerfile.in.lib.gpu index c7c86a2cf8cd..03b920a685ff 100644 --- a/docker/Dockerfiles/Dockerfile.in.lib.gpu +++ b/docker/Dockerfiles/Dockerfile.in.lib.gpu @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # dockerfile to build libmxnet.so on GPU FROM nvidia/cuda:8.0-cudnn5-devel diff --git a/docker/Dockerfiles/Dockerfile.in.perl b/docker/Dockerfiles/Dockerfile.in.perl index 14b4f0194d6e..6e801dcc48c3 100644 --- a/docker/Dockerfiles/Dockerfile.in.perl +++ b/docker/Dockerfiles/Dockerfile.in.perl @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # part of the dockerfile to install the perl binding COPY install/perl.sh install/ diff --git a/docker/Dockerfiles/Dockerfile.in.python b/docker/Dockerfiles/Dockerfile.in.python index b7979b231d7d..aa8bb2748d7f 100644 --- a/docker/Dockerfiles/Dockerfile.in.python +++ b/docker/Dockerfiles/Dockerfile.in.python @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # part of the dockerfile to install the python binding COPY install/python.sh install/ diff --git a/docker/Dockerfiles/Dockerfile.in.r-lang b/docker/Dockerfiles/Dockerfile.in.r-lang index 321094ec6c63..33f6f3533c18 100644 --- a/docker/Dockerfiles/Dockerfile.in.r-lang +++ b/docker/Dockerfiles/Dockerfile.in.r-lang @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # part of the dockerfile to install the r binding COPY install/r.sh install/ diff --git a/docker/Dockerfiles/Dockerfile.in.scala b/docker/Dockerfiles/Dockerfile.in.scala index 92e098384e13..ee1b96240a25 100644 --- a/docker/Dockerfiles/Dockerfile.in.scala +++ b/docker/Dockerfiles/Dockerfile.in.scala @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # part of the dockerfile to install the scala binding COPY install/scala.sh install/ diff --git a/docker/Dockerfiles/Dockerfile.tensorrt b/docker/Dockerfiles/Dockerfile.tensorrt index 2e95a7618360..f5603e7c1e59 100644 --- a/docker/Dockerfiles/Dockerfile.tensorrt +++ b/docker/Dockerfiles/Dockerfile.tensorrt @@ -1,4 +1,5 @@ # -*- mode: dockerfile -*- +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,6 +17,7 @@ # specific language governing permissions and limitations # under the License. # +# # Dockerfile to run MXNet with TensorRT Integration. FROM nvidia/cuda:9.0-cudnn7-devel diff --git a/docker/Dockerfiles/License.md b/docker/Dockerfiles/License.md deleted file mode 100644 index 614c582aefa8..000000000000 --- a/docker/Dockerfiles/License.md +++ /dev/null @@ -1,20 +0,0 @@ -## Dockerfiles License - -### All Dockerfiles in this directory are provided under an Apache 2.0 license. - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index 971a2de7894c..3aa50eccd89d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,3 +1,22 @@ + + # Docker images for MXNET ## How to use diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index f806d5d6f459..6d17c50439ae 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -1,38 +1,57 @@ + + # Release Python Docker Images for MXNet -The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release. +The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release. It uses the appropriate pip binaries to build different docker images as - * cpu * cpu_mkl * latest (same as cpu) -* gpu_cu90 +* gpu_cu90 * gpu_cu90_mkl * gpu (same as gpu_cu90) -* gpu_cu80 +* gpu_cu80 * gpu_cu80_mkl -* gpu_cu92 +* gpu_cu92 * gpu_cu92_mkl -** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required. +** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required. Refer: https://pypi.org/project/mxnet/ ### Usage `./build_python_dockerfile.sh ` -For example: +For example: `./build_python_dockerfile.sh 1.3.0 ~/build-docker/incubator-mxnet` -** Note: The build script picks up the latest pip binaries. This means it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly. +** Note: The build script picks up the latest pip binaries. This means it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly. ### Tests run * [test_conv.py](https://github.com/apache/incubator-mxnet/blob/master/tests/python/train/test_conv.py) * [train_mnist.py](https://github.com/apache/incubator-mxnet/blob/master/example/image-classification/train_mnist.py) -* [test_mxnet.py](https://github.com/apache/incubator-mxnet/blob/master/docker/docker-python/test_mxnet.py): This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter. +* [test_mxnet.py](https://github.com/apache/incubator-mxnet/blob/master/docker/docker-python/test_mxnet.py): This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter. ### Dockerhub Credentials -Dockerhub credentials will be required to push images at the end of this script. +Dockerhub credentials will be required to push images at the end of this script. Credentials can be provided in the following ways: * **Interactive Login:** Run the script as is and it will ask you for credentials interactively. * **Be Already Logged in:** Login to the mxnet dockerhub account before you run the build script and the script will complete build, test and push. diff --git a/docs/Dockerfile b/docs/Dockerfile index 99bb3d5be492..61a1e5f915e2 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,3 +1,23 @@ +# -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# FROM ubuntu:14.04 MAINTAINER Mu Li diff --git a/docs/README.md b/docs/README.md index 80463cc68d54..6ff2ae5efe18 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,3 +1,22 @@ + + # Building and Updating MXNet Documentation The website is hosted at http://mxnet.incubator.apache.org/. diff --git a/docs/api/c++/index.md b/docs/api/c++/index.md index 6bb7a410b341..491e91ab5e5d 100644 --- a/docs/api/c++/index.md +++ b/docs/api/c++/index.md @@ -1,3 +1,22 @@ + + # MXNet - C++ API The MXNet C++ Package provides C++ API bindings to the users of MXNet. Currently, these bindings are not available as standalone package. diff --git a/docs/api/clojure/index.md b/docs/api/clojure/index.md index 32abbe06ad79..cb27c637d586 100644 --- a/docs/api/clojure/index.md +++ b/docs/api/clojure/index.md @@ -1,3 +1,22 @@ + + # MXNet - Clojure API MXNet supports the Clojure programming language. The MXNet Clojure package brings flexible and efficient GPU diff --git a/docs/api/clojure/kvstore.md b/docs/api/clojure/kvstore.md index 25943c365f94..d5f7c089ff7c 100644 --- a/docs/api/clojure/kvstore.md +++ b/docs/api/clojure/kvstore.md @@ -1,3 +1,22 @@ + + # KVStore API Topics: diff --git a/docs/api/clojure/module.md b/docs/api/clojure/module.md index 88a5308a7cc2..db7cb70d0fce 100644 --- a/docs/api/clojure/module.md +++ b/docs/api/clojure/module.md @@ -1,3 +1,22 @@ + + # Module API The module API provides an intermediate and high-level interface for performing computation with neural networks in MXNet. Module wraps a Symbol and one or more Executors. It has both a high level and intermediate level API. diff --git a/docs/api/clojure/ndarray.md b/docs/api/clojure/ndarray.md index b0e5c991f7d4..3d367c6c5011 100644 --- a/docs/api/clojure/ndarray.md +++ b/docs/api/clojure/ndarray.md @@ -1,3 +1,22 @@ + + # NDArray API diff --git a/docs/api/clojure/symbol.md b/docs/api/clojure/symbol.md index 1ec841f153cf..a70c9ea5b906 100644 --- a/docs/api/clojure/symbol.md +++ b/docs/api/clojure/symbol.md @@ -1,3 +1,22 @@ + + # MXNet Clojure Symbolic API Topics: diff --git a/docs/api/clojure/symbol_in_pictures.md b/docs/api/clojure/symbol_in_pictures.md index bde274cb238b..aac133d2d067 100644 --- a/docs/api/clojure/symbol_in_pictures.md +++ b/docs/api/clojure/symbol_in_pictures.md @@ -1,3 +1,22 @@ + + # Symbolic Configuration and Execution in Pictures This topic explains symbolic construction and execution in pictures. diff --git a/docs/api/index.md b/docs/api/index.md index 9e7a58f7778c..62de6cbed502 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -1,3 +1,22 @@ + + # MXNet APIs diff --git a/docs/api/java/index.md b/docs/api/java/index.md index b2ae035b19c9..9955c05f3bc1 100644 --- a/docs/api/java/index.md +++ b/docs/api/java/index.md @@ -1,3 +1,22 @@ + + # MXNet - Java Inference API MXNet supports Java for performing inference on a trained model. The MXNet Java Inference API is an extension of the [Scala Infer API](../../api/scala/infer.html) which provides model loading and inference functionality. diff --git a/docs/api/julia/index.md b/docs/api/julia/index.md index f4dc0e5b7498..0c5c93c9c4db 100644 --- a/docs/api/julia/index.md +++ b/docs/api/julia/index.md @@ -1,3 +1,22 @@ + + # MXNet - Julia API See the [MXNet Julia Reference Manual](https://media.readthedocs.org/pdf/mxnet-test/latest/mxnet-test.pdf). diff --git a/docs/api/perl/index.md b/docs/api/perl/index.md index 6408b525f00c..0aef4d4aa4d3 100644 --- a/docs/api/perl/index.md +++ b/docs/api/perl/index.md @@ -1,3 +1,22 @@ + + # MXNet - Perl API MXNet supports the Perl programming language. The MXNet Perl package brings flexible and efficient GPU diff --git a/docs/api/perl/io.md b/docs/api/perl/io.md index 3310f26aba18..c3f55c6e27ea 100644 --- a/docs/api/perl/io.md +++ b/docs/api/perl/io.md @@ -1,3 +1,22 @@ + + # Data Loading API ## Overview diff --git a/docs/api/perl/kvstore.md b/docs/api/perl/kvstore.md index 7ff73be06b07..e221dbbf4f9b 100644 --- a/docs/api/perl/kvstore.md +++ b/docs/api/perl/kvstore.md @@ -1,3 +1,22 @@ + + # KVStore API Topics: diff --git a/docs/api/perl/module.md b/docs/api/perl/module.md index 132e333dac9c..314b5e8d41da 100644 --- a/docs/api/perl/module.md +++ b/docs/api/perl/module.md @@ -1,3 +1,22 @@ + + # Module API ## Overview diff --git a/docs/api/perl/ndarray.md b/docs/api/perl/ndarray.md index c7962b9f896f..4e7e3a3562ce 100644 --- a/docs/api/perl/ndarray.md +++ b/docs/api/perl/ndarray.md @@ -1,3 +1,22 @@ + + # NDArray API ## Overview diff --git a/docs/api/perl/symbol.md b/docs/api/perl/symbol.md index 407e912d6fb5..d9d7d7143c26 100644 --- a/docs/api/perl/symbol.md +++ b/docs/api/perl/symbol.md @@ -1,3 +1,22 @@ + + # MXNet Perl Symbolic API Topics: diff --git a/docs/api/python/autograd/autograd.md b/docs/api/python/autograd/autograd.md index f20799a44351..38a99f504f60 100644 --- a/docs/api/python/autograd/autograd.md +++ b/docs/api/python/autograd/autograd.md @@ -1,3 +1,22 @@ + + # Autograd Package ```eval_rst diff --git a/docs/api/python/callback/callback.md b/docs/api/python/callback/callback.md index 714966678a15..be99dc01ee7b 100644 --- a/docs/api/python/callback/callback.md +++ b/docs/api/python/callback/callback.md @@ -1,3 +1,22 @@ + + # Callback API ```eval_rst diff --git a/docs/api/python/contrib/contrib.md b/docs/api/python/contrib/contrib.md index 66fc39152983..65922556d2cc 100644 --- a/docs/api/python/contrib/contrib.md +++ b/docs/api/python/contrib/contrib.md @@ -1,3 +1,22 @@ + + # Contrib Package ## Overview diff --git a/docs/api/python/contrib/onnx.md b/docs/api/python/contrib/onnx.md index f8210ad6a001..18b108caa5b5 100644 --- a/docs/api/python/contrib/onnx.md +++ b/docs/api/python/contrib/onnx.md @@ -1,3 +1,22 @@ + + # ONNX-MXNet API ## Overview diff --git a/docs/api/python/contrib/svrg_optimization.md b/docs/api/python/contrib/svrg_optimization.md index e6e1c3e23ee3..c79dd44f003b 100644 --- a/docs/api/python/contrib/svrg_optimization.md +++ b/docs/api/python/contrib/svrg_optimization.md @@ -1,3 +1,22 @@ + + # SVRG Optimization in Python Module API ## Overview diff --git a/docs/api/python/contrib/text.md b/docs/api/python/contrib/text.md index e5f92bc7ef01..1fd9514e0f65 100644 --- a/docs/api/python/contrib/text.md +++ b/docs/api/python/contrib/text.md @@ -1,3 +1,22 @@ + + # Text API ## Overview diff --git a/docs/api/python/executor/executor.md b/docs/api/python/executor/executor.md index a37bfa91d2bf..26c97907dfc5 100644 --- a/docs/api/python/executor/executor.md +++ b/docs/api/python/executor/executor.md @@ -1,3 +1,22 @@ + + # Executor and Executor Manager The executor and executor manager are internal classes for managing symbolic diff --git a/docs/api/python/gluon/contrib.md b/docs/api/python/gluon/contrib.md index 98f36f882164..f6e6dfe771a5 100644 --- a/docs/api/python/gluon/contrib.md +++ b/docs/api/python/gluon/contrib.md @@ -1,3 +1,22 @@ + + # Gluon Contrib API ## Overview diff --git a/docs/api/python/gluon/data.md b/docs/api/python/gluon/data.md index 99e54c1f034b..6d4440f91709 100644 --- a/docs/api/python/gluon/data.md +++ b/docs/api/python/gluon/data.md @@ -1,3 +1,22 @@ + + # Gluon Data API ## Overview diff --git a/docs/api/python/gluon/gluon.md b/docs/api/python/gluon/gluon.md index 07c98981a166..cbb0750e47a2 100644 --- a/docs/api/python/gluon/gluon.md +++ b/docs/api/python/gluon/gluon.md @@ -1,3 +1,22 @@ + + # Gluon Package diff --git a/docs/api/python/gluon/loss.md b/docs/api/python/gluon/loss.md index 948f4983d370..a2ac65f84b53 100644 --- a/docs/api/python/gluon/loss.md +++ b/docs/api/python/gluon/loss.md @@ -1,3 +1,22 @@ + + # Gluon Loss API ## Overview diff --git a/docs/api/python/gluon/model_zoo.md b/docs/api/python/gluon/model_zoo.md index 288cf96f1d09..07d11cd7242e 100644 --- a/docs/api/python/gluon/model_zoo.md +++ b/docs/api/python/gluon/model_zoo.md @@ -1,3 +1,22 @@ + + # Gluon Model Zoo ```eval_rst diff --git a/docs/api/python/gluon/nn.md b/docs/api/python/gluon/nn.md index 25c82f066680..544a315eb784 100644 --- a/docs/api/python/gluon/nn.md +++ b/docs/api/python/gluon/nn.md @@ -1,3 +1,22 @@ + + # Gluon Neural Network Layers ## Overview diff --git a/docs/api/python/gluon/rnn.md b/docs/api/python/gluon/rnn.md index b558b8479e43..e7da65741f19 100644 --- a/docs/api/python/gluon/rnn.md +++ b/docs/api/python/gluon/rnn.md @@ -1,3 +1,22 @@ + + # Gluon Recurrent Neural Network API ## Overview diff --git a/docs/api/python/image/image.md b/docs/api/python/image/image.md index 299c54570b17..61cca225a230 100644 --- a/docs/api/python/image/image.md +++ b/docs/api/python/image/image.md @@ -1,3 +1,22 @@ + + # Image API ## Overview diff --git a/docs/api/python/index.md b/docs/api/python/index.md index de86aedff691..1d11eb7182a1 100644 --- a/docs/api/python/index.md +++ b/docs/api/python/index.md @@ -1,3 +1,22 @@ + + # MXNet - Python API MXNet provides a comprehensive and flexible Python API to serve a broad community of developers with different levels of experience and wide ranging requirements. In this section, we provide an in-depth discussion of the functionality provided by various MXNet Python packages. diff --git a/docs/api/python/io/io.md b/docs/api/python/io/io.md index a069eae61e71..057d64c129a4 100644 --- a/docs/api/python/io/io.md +++ b/docs/api/python/io/io.md @@ -1,3 +1,22 @@ + + # Data Loading API ## Overview diff --git a/docs/api/python/kvstore/kvstore.md b/docs/api/python/kvstore/kvstore.md index 9f4c06499663..3a2957353620 100644 --- a/docs/api/python/kvstore/kvstore.md +++ b/docs/api/python/kvstore/kvstore.md @@ -1,3 +1,22 @@ + + # KVStore API .. note:: Direct interactions with ``KVStore`` are dangerous and not recommended. diff --git a/docs/api/python/metric/metric.md b/docs/api/python/metric/metric.md index 319647a69491..5423971ad6fb 100644 --- a/docs/api/python/metric/metric.md +++ b/docs/api/python/metric/metric.md @@ -1,3 +1,22 @@ + + # Evaluation Metric API ```eval_rst diff --git a/docs/api/python/model.md b/docs/api/python/model.md index dc36648f7fac..a98d2e4119db 100644 --- a/docs/api/python/model.md +++ b/docs/api/python/model.md @@ -1,3 +1,22 @@ + + # Model API The model API provides a simplified way to train neural networks using common best practices. diff --git a/docs/api/python/module/module.md b/docs/api/python/module/module.md index 7a86eccd8d3a..51c2343deb58 100644 --- a/docs/api/python/module/module.md +++ b/docs/api/python/module/module.md @@ -1,3 +1,22 @@ + + # Module API ```eval_rst diff --git a/docs/api/python/ndarray/contrib.md b/docs/api/python/ndarray/contrib.md index d7c9021b5957..8d37ca41f4ba 100644 --- a/docs/api/python/ndarray/contrib.md +++ b/docs/api/python/ndarray/contrib.md @@ -1,3 +1,22 @@ + + # Contrib NDArray API ```eval_rst diff --git a/docs/api/python/ndarray/linalg.md b/docs/api/python/ndarray/linalg.md index 0a85b48304fa..fac44c0f06ef 100644 --- a/docs/api/python/ndarray/linalg.md +++ b/docs/api/python/ndarray/linalg.md @@ -1,3 +1,22 @@ + + # Linear Algebra NDArray API ```eval_rst diff --git a/docs/api/python/ndarray/ndarray.md b/docs/api/python/ndarray/ndarray.md index 2df18c286ba7..b71b961fef3c 100644 --- a/docs/api/python/ndarray/ndarray.md +++ b/docs/api/python/ndarray/ndarray.md @@ -1,3 +1,22 @@ + + # NDArray API ```eval_rst diff --git a/docs/api/python/ndarray/random.md b/docs/api/python/ndarray/random.md index 60c565dd5528..fc8c25900415 100644 --- a/docs/api/python/ndarray/random.md +++ b/docs/api/python/ndarray/random.md @@ -1,3 +1,22 @@ + + # Random Distribution Generator NDArray API ```eval_rst diff --git a/docs/api/python/ndarray/sparse.md b/docs/api/python/ndarray/sparse.md index 608a9501a039..0a513dc7a624 100644 --- a/docs/api/python/ndarray/sparse.md +++ b/docs/api/python/ndarray/sparse.md @@ -1,3 +1,22 @@ + + # Sparse NDArray API ```eval_rst diff --git a/docs/api/python/optimization/contrib.md b/docs/api/python/optimization/contrib.md index 8fc261f4f052..d67887992e50 100644 --- a/docs/api/python/optimization/contrib.md +++ b/docs/api/python/optimization/contrib.md @@ -1,3 +1,22 @@ + + # Contrib Optimization API ```eval_rst diff --git a/docs/api/python/optimization/optimization.md b/docs/api/python/optimization/optimization.md index fa3547b54a8f..56bf8f538ed8 100644 --- a/docs/api/python/optimization/optimization.md +++ b/docs/api/python/optimization/optimization.md @@ -1,3 +1,22 @@ + + # Optimization: initialize and update weights ## Overview diff --git a/docs/api/python/rtc/rtc.md b/docs/api/python/rtc/rtc.md index 88e4ed892722..5f90f3a8b77c 100644 --- a/docs/api/python/rtc/rtc.md +++ b/docs/api/python/rtc/rtc.md @@ -1,3 +1,22 @@ + + # Run-Time Compilation API ```eval_rst diff --git a/docs/api/python/symbol/contrib.md b/docs/api/python/symbol/contrib.md index a0253216f945..2429b3765ace 100644 --- a/docs/api/python/symbol/contrib.md +++ b/docs/api/python/symbol/contrib.md @@ -1,3 +1,22 @@ + + # Contrib Symbol API ```eval_rst diff --git a/docs/api/python/symbol/linalg.md b/docs/api/python/symbol/linalg.md index d22ca8e171f9..c04846e7ba92 100644 --- a/docs/api/python/symbol/linalg.md +++ b/docs/api/python/symbol/linalg.md @@ -1,3 +1,22 @@ + + # Linear Algebra Symbol API ```eval_rst diff --git a/docs/api/python/symbol/random.md b/docs/api/python/symbol/random.md index 1ecaf38830fc..1c5f916ad10a 100644 --- a/docs/api/python/symbol/random.md +++ b/docs/api/python/symbol/random.md @@ -1,3 +1,22 @@ + + # Random Distribution Generator Symbol API ```eval_rst diff --git a/docs/api/python/symbol/rnn.md b/docs/api/python/symbol/rnn.md index 722180a27789..627d2e0cde37 100644 --- a/docs/api/python/symbol/rnn.md +++ b/docs/api/python/symbol/rnn.md @@ -1,3 +1,22 @@ + + # RNN Cell Symbol API ```eval_rst diff --git a/docs/api/python/symbol/sparse.md b/docs/api/python/symbol/sparse.md index cd8272cedd7d..866a302940cc 100644 --- a/docs/api/python/symbol/sparse.md +++ b/docs/api/python/symbol/sparse.md @@ -1,3 +1,22 @@ + + # Sparse Symbol API ```eval_rst diff --git a/docs/api/python/symbol/symbol.md b/docs/api/python/symbol/symbol.md index 0fc2aa7c6cf2..9783053d9a1d 100644 --- a/docs/api/python/symbol/symbol.md +++ b/docs/api/python/symbol/symbol.md @@ -1,3 +1,22 @@ + + # Symbol API ```eval_rst diff --git a/docs/api/python/symbol_in_pictures/symbol_in_pictures.md b/docs/api/python/symbol_in_pictures/symbol_in_pictures.md index 5ef4395b9af6..5326ade3003f 100644 --- a/docs/api/python/symbol_in_pictures/symbol_in_pictures.md +++ b/docs/api/python/symbol_in_pictures/symbol_in_pictures.md @@ -1,3 +1,22 @@ + + # Symbolic Configuration and Execution in Pictures This topic explains symbolic construction and execution in pictures. diff --git a/docs/api/python/tools/test_utils.md b/docs/api/python/tools/test_utils.md index e29d9b99a844..2896a594c004 100644 --- a/docs/api/python/tools/test_utils.md +++ b/docs/api/python/tools/test_utils.md @@ -1,3 +1,22 @@ + + # Test Utilities This module has a variety of tools that help using and testing MXNet. diff --git a/docs/api/python/tools/visualization.md b/docs/api/python/tools/visualization.md index 6faa24980dd8..8ba722878616 100644 --- a/docs/api/python/tools/visualization.md +++ b/docs/api/python/tools/visualization.md @@ -1,3 +1,22 @@ + + # Visualization This module contains visualization features. diff --git a/docs/api/r/index.md b/docs/api/r/index.md index 822c59dbe3cc..5a85a770d80e 100644 --- a/docs/api/r/index.md +++ b/docs/api/r/index.md @@ -1,3 +1,22 @@ + + # MXNet - R API See the [MXNet R Reference Manual](https://s3.amazonaws.com/mxnet-prod/docs/R/mxnet-r-reference-manual.pdf). diff --git a/docs/api/scala/index.md b/docs/api/scala/index.md index f7a150019872..76e5673aa3bf 100644 --- a/docs/api/scala/index.md +++ b/docs/api/scala/index.md @@ -1,3 +1,22 @@ + + # MXNet - Scala API MXNet supports the Scala programming language. The MXNet Scala package brings flexible and efficient GPU diff --git a/docs/api/scala/infer.md b/docs/api/scala/infer.md index dfa21c178416..af50fc2e4b03 100644 --- a/docs/api/scala/infer.md +++ b/docs/api/scala/infer.md @@ -1,3 +1,22 @@ + + # Infer API The MXNet Scala Infer API provides you with model loading and inference functionality using the MXNet Scala package. diff --git a/docs/api/scala/io.md b/docs/api/scala/io.md index ff212d1479ed..fe819e876bbd 100644 --- a/docs/api/scala/io.md +++ b/docs/api/scala/io.md @@ -1,3 +1,22 @@ + + # MXNet Scala Data Loading API This topic introduces the data input method for MXNet. MXNet uses an iterator to provide data to the neural network. Iterators do some preprocessing and generate batches for the neural network. diff --git a/docs/api/scala/kvstore.md b/docs/api/scala/kvstore.md index e195c4d7e720..32e3c1f1145c 100644 --- a/docs/api/scala/kvstore.md +++ b/docs/api/scala/kvstore.md @@ -1,3 +1,22 @@ + + # KVStore API Topics: diff --git a/docs/api/scala/model.md b/docs/api/scala/model.md index 3e09d15218b8..4eeae42de6be 100644 --- a/docs/api/scala/model.md +++ b/docs/api/scala/model.md @@ -1,3 +1,22 @@ + + # MXNet Scala Model API The model API provides a simplified way to train neural networks using common best practices. diff --git a/docs/api/scala/module.md b/docs/api/scala/module.md index 06487f734a3f..c7f9a0a29a3b 100644 --- a/docs/api/scala/module.md +++ b/docs/api/scala/module.md @@ -1,3 +1,22 @@ + + # Module API The module API provides an intermediate and high-level interface for performing computation with neural networks in MXNet. A *module* is an instance of subclasses of the `BaseModule`. The most widely used module class is called `Module`. Module wraps a `Symbol` and one or more `Executors`. For a full list of functions, see `BaseModule`. A subclass of modules might have extra interface functions. This topic provides some examples of common use cases. All of the module APIs are in the `Module` namespace. diff --git a/docs/api/scala/ndarray.md b/docs/api/scala/ndarray.md index 9e87d397c8b8..ad3349d02c8e 100644 --- a/docs/api/scala/ndarray.md +++ b/docs/api/scala/ndarray.md @@ -1,3 +1,22 @@ + + # NDArray API diff --git a/docs/api/scala/symbol.md b/docs/api/scala/symbol.md index c10d5fb60a24..52d63e91f38b 100644 --- a/docs/api/scala/symbol.md +++ b/docs/api/scala/symbol.md @@ -1,3 +1,22 @@ + + # MXNet Scala Symbolic API Topics: diff --git a/docs/api/scala/symbol_in_pictures.md b/docs/api/scala/symbol_in_pictures.md index 0fa12d0a7557..9d6ecdd55202 100644 --- a/docs/api/scala/symbol_in_pictures.md +++ b/docs/api/scala/symbol_in_pictures.md @@ -1,3 +1,22 @@ + + # Symbolic Configuration and Execution in Pictures This topic explains symbolic construction and execution in pictures. diff --git a/docs/architecture/exception_handling.md b/docs/architecture/exception_handling.md index 5b4448a53ab6..08b05058628e 100644 --- a/docs/architecture/exception_handling.md +++ b/docs/architecture/exception_handling.md @@ -1,3 +1,22 @@ + + # Exception Handling in MXNet This tutorial explains the exception handling support in MXNet, diff --git a/docs/architecture/index.md b/docs/architecture/index.md index 189e76e62fa5..d17564090c89 100644 --- a/docs/architecture/index.md +++ b/docs/architecture/index.md @@ -1,3 +1,22 @@ + + # MXNet Architecture Building a high-performance deep learning library diff --git a/docs/architecture/note_data_loading.md b/docs/architecture/note_data_loading.md index 7c92c86a9c67..83420a3ff071 100644 --- a/docs/architecture/note_data_loading.md +++ b/docs/architecture/note_data_loading.md @@ -1,3 +1,22 @@ + + # Designing Efficient Data Loaders for Deep Learning Data loading is an important component of any machine learning system. diff --git a/docs/architecture/note_engine.md b/docs/architecture/note_engine.md index 237d8d61b77d..378191cdb184 100644 --- a/docs/architecture/note_engine.md +++ b/docs/architecture/note_engine.md @@ -1,3 +1,22 @@ + + # Dependency Engine for Deep Learning We always want deep learning libraries diff --git a/docs/architecture/note_memory.md b/docs/architecture/note_memory.md index 8ec77c395f31..cd161c9ed0f0 100644 --- a/docs/architecture/note_memory.md +++ b/docs/architecture/note_memory.md @@ -1,3 +1,22 @@ + + # Optimizing Memory Consumption in Deep Learning Over the last ten years, a constant trend in deep learning diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 6a37f8830479..04abacd869cd 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -1,3 +1,22 @@ + + # MXNet System Architecture ![System Overview](https://raw.githubusercontent.com/dmlc/dmlc.github.io/master/img/mxnet/system/overview.png) diff --git a/docs/architecture/program_model.md b/docs/architecture/program_model.md index 519a9a9024d8..b4d2c31accda 100644 --- a/docs/architecture/program_model.md +++ b/docs/architecture/program_model.md @@ -1,3 +1,22 @@ + + # Deep Learning Programming Style However much we might ultimately care about performance, diff --git a/docs/architecture/rnn_interface.md b/docs/architecture/rnn_interface.md index dc0b6a7958ef..523295e934df 100644 --- a/docs/architecture/rnn_interface.md +++ b/docs/architecture/rnn_interface.md @@ -1,3 +1,22 @@ + + # Survey of Existing Interfaces and Implementations Commonly used deep learning libraries with good RNN/LSTM support include [Theano](http://deeplearning.net/software/theano/library/scan.html) and its wrappers [Lasagne](http://lasagne.readthedocs.org/en/latest/modules/layers/recurrent.html) and [Keras](http://keras.io/layers/recurrent/); [CNTK](https://cntk.codeplex.com/); [TensorFlow](https://www.tensorflow.org/tutorials/sequences/recurrent); and various implementations in Torch, such as [this well-known character-level language model tutorial](https://github.com/karpathy/char-rnn), [this](https://github.com/Element-Research/rnn). diff --git a/docs/build_version_doc/Dockerfile b/docs/build_version_doc/Dockerfile index 0cc2680ffaef..8bae701199af 100755 --- a/docs/build_version_doc/Dockerfile +++ b/docs/build_version_doc/Dockerfile @@ -1,3 +1,23 @@ +# -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# FROM ubuntu:16.04 LABEL maintainer="markhama@amazon.com" diff --git a/docs/build_version_doc/README.md b/docs/build_version_doc/README.md index 50802371cea2..2e30a3bbe5f9 100644 --- a/docs/build_version_doc/README.md +++ b/docs/build_version_doc/README.md @@ -1,3 +1,22 @@ + + # MXNet Documentation Build Scripts This folder contains a variety of scripts to generate the MXNet.io website as well as the docs for different versions of MXNet. diff --git a/docs/community/contribute.md b/docs/community/contribute.md index 1b83638edcb5..727b9d137784 100644 --- a/docs/community/contribute.md +++ b/docs/community/contribute.md @@ -1,3 +1,22 @@ + + # Contributing to MXNet Apache MXNet (incubating) is a community led, open source deep learning project. We welcome new members and look forward to your contributions. Here you will find how to get started and links to detailed information on MXNet best practices and processes. diff --git a/docs/community/ecosystem.md b/docs/community/ecosystem.md index 100ed973cf3f..792f9de6ff3c 100644 --- a/docs/community/ecosystem.md +++ b/docs/community/ecosystem.md @@ -1,3 +1,22 @@ + + # MXNet's Ecosystem Community contributions to MXNet have added many new valuable features and functionality to support use cases such as model serving & portability, easy and flexible APIs, and educational material like crash courses and online books. This ecosystem page lists the projects that use MXNet, teach MXNet, or augment MXNet in some way. diff --git a/docs/community/index.md b/docs/community/index.md index 7bdb1c213503..a2be9861db96 100644 --- a/docs/community/index.md +++ b/docs/community/index.md @@ -1,3 +1,22 @@ + + # MXNet Community ```eval_rst diff --git a/docs/community/mxnet_channels.md b/docs/community/mxnet_channels.md index 8bc8471dffd1..ec0149769474 100644 --- a/docs/community/mxnet_channels.md +++ b/docs/community/mxnet_channels.md @@ -1,3 +1,22 @@ + + # Join MXNet Development Discussion Converse with the MXNet community via the following channels: diff --git a/docs/community/powered_by.md b/docs/community/powered_by.md index 82748ef7df2a..0cca3d02f0c1 100644 --- a/docs/community/powered_by.md +++ b/docs/community/powered_by.md @@ -1,3 +1,22 @@ + + # Powered By
diff --git a/docs/faq/add_op_in_backend.md b/docs/faq/add_op_in_backend.md index c44a0aa05235..64f6c8ae0f98 100644 --- a/docs/faq/add_op_in_backend.md +++ b/docs/faq/add_op_in_backend.md @@ -1,3 +1,22 @@ + + # A Beginner's Guide to Implementing Operators in MXNet Backend ## Introduction diff --git a/docs/faq/bucketing.md b/docs/faq/bucketing.md index dbfdedde2acf..49994dc4fe97 100644 --- a/docs/faq/bucketing.md +++ b/docs/faq/bucketing.md @@ -1,3 +1,22 @@ + + # Bucketing in MXNet When we train recurrent neural networks (RNNs), we _unroll_ the network in time. For a single example of length T, we would unroll the network T steps. diff --git a/docs/faq/caffe.md b/docs/faq/caffe.md index a9bab3cdb549..b99acb07e0bd 100644 --- a/docs/faq/caffe.md +++ b/docs/faq/caffe.md @@ -1,3 +1,22 @@ + + # How to | Convert from Caffe to MXNet Key topics covered include the following: diff --git a/docs/faq/cloud.md b/docs/faq/cloud.md index 67b28f8b4338..bd2987f0a057 100644 --- a/docs/faq/cloud.md +++ b/docs/faq/cloud.md @@ -1,3 +1,22 @@ + + # MXNet on the Cloud Deep learning can require extremely powerful hardware, often for unpredictable durations of time. diff --git a/docs/faq/develop_and_hack.md b/docs/faq/develop_and_hack.md index da53fd010bb0..b09824dc2ef6 100644 --- a/docs/faq/develop_and_hack.md +++ b/docs/faq/develop_and_hack.md @@ -1,3 +1,22 @@ + + # Develop and Hack MXNet - [Create new operators](new_op.md) - [Use Torch from MXNet](torch.md) diff --git a/docs/faq/distributed_training.md b/docs/faq/distributed_training.md index 8d8666ff066c..32696f34e250 100644 --- a/docs/faq/distributed_training.md +++ b/docs/faq/distributed_training.md @@ -1,3 +1,22 @@ + + # Distributed Training in MXNet MXNet supports distributed training enabling us to leverage multiple machines for faster training. In this document, we describe how it works, how to launch a distributed training job and diff --git a/docs/faq/env_var.md b/docs/faq/env_var.md index 83368bf4d0c3..2f649bedafc1 100644 --- a/docs/faq/env_var.md +++ b/docs/faq/env_var.md @@ -1,3 +1,22 @@ + + Environment Variables ===================== MXNet has several settings that you can change with environment variables. diff --git a/docs/faq/faq.md b/docs/faq/faq.md index 668587ec6888..4682c95d2a47 100644 --- a/docs/faq/faq.md +++ b/docs/faq/faq.md @@ -1,3 +1,22 @@ + + # Frequently Asked Questions This topic provides answers to the frequently asked questions on [mxnet/issues](https://github.com/dmlc/mxnet/issues). Before posting an issue, please check this page. If you would like to contribute to this page, please make the questions and answers simple. If your answer is extremely detailed, please post it elsewhere and link to it. diff --git a/docs/faq/finetune.md b/docs/faq/finetune.md index 04244d15b0b9..33a4ffb32521 100644 --- a/docs/faq/finetune.md +++ b/docs/faq/finetune.md @@ -1,3 +1,22 @@ + + # Fine-tune with Pretrained Models diff --git a/docs/faq/float16.md b/docs/faq/float16.md index b4cd97b30e5c..0fda178c4bde 100644 --- a/docs/faq/float16.md +++ b/docs/faq/float16.md @@ -1,3 +1,22 @@ + + # Mixed precision training using float16 In this tutorial you will walk through how one can train deep learning neural networks with mixed precision on supported hardware. You will first see how to use float16 (both with Gluon and Symbolic APIs) and then some techniques on achieving good performance and accuracy. diff --git a/docs/faq/gradient_compression.md b/docs/faq/gradient_compression.md index e2dbd3271d8d..adf7f62bc193 100644 --- a/docs/faq/gradient_compression.md +++ b/docs/faq/gradient_compression.md @@ -1,3 +1,22 @@ + + # Gradient Compression Gradient Compression reduces communication bandwidth, and in some scenarios, it can make training more scalable and efficient without significant loss in convergence rate or accuracy. Example implementations with GPUs, CPUs, and distributed training are provided in this document. diff --git a/docs/faq/index.md b/docs/faq/index.md index fe91f7ca43b7..99c448da027d 100644 --- a/docs/faq/index.md +++ b/docs/faq/index.md @@ -1,3 +1,22 @@ + + # MXNet FAQ ```eval_rst diff --git a/docs/faq/model_parallel_lstm.md b/docs/faq/model_parallel_lstm.md index b78b2c574dcc..63f4db6d9df9 100644 --- a/docs/faq/model_parallel_lstm.md +++ b/docs/faq/model_parallel_lstm.md @@ -1,3 +1,22 @@ + + # Training with Multiple GPUs Using Model Parallelism Training deep learning models can be resource intensive. Even with a powerful GPU, some models can take days or weeks to train. diff --git a/docs/faq/multi_devices.md b/docs/faq/multi_devices.md index a43879cb5233..46ec837025d9 100644 --- a/docs/faq/multi_devices.md +++ b/docs/faq/multi_devices.md @@ -1,3 +1,22 @@ + + # Run MXNet on Multiple CPU/GPUs with Data Parallelism _MXNet_ supports training with multiple CPUs and GPUs, which may be located on different physical machines. diff --git a/docs/faq/new_op.md b/docs/faq/new_op.md index 994a2a6f823e..910bb64ef5d9 100644 --- a/docs/faq/new_op.md +++ b/docs/faq/new_op.md @@ -1,3 +1,22 @@ + + # How to Create New Operators (Layers) This tutorials walks you through the process of creating new MXNet operators (or layers). diff --git a/docs/faq/nnpack.md b/docs/faq/nnpack.md index ed38cb07df7e..690315efe54a 100644 --- a/docs/faq/nnpack.md +++ b/docs/faq/nnpack.md @@ -1,3 +1,22 @@ + + ### NNPACK for Multi-Core CPU Support in MXNet [NNPACK](https://github.com/Maratyszcza/NNPACK) is an acceleration package for neural network computations, which can run on x86-64, ARMv7, or ARM64 architecture CPUs. diff --git a/docs/faq/perf.md b/docs/faq/perf.md index f116ede11d56..d00c904c0873 100644 --- a/docs/faq/perf.md +++ b/docs/faq/perf.md @@ -1,3 +1,22 @@ + + # Some Tips for Improving MXNet Performance Even after fixing the training or deployment environment and parallelization scheme, a number of configuration settings and data-handling choices can impact the _MXNet_ performance. diff --git a/docs/faq/recordio.md b/docs/faq/recordio.md index 3091052ef6f3..3ece38617513 100644 --- a/docs/faq/recordio.md +++ b/docs/faq/recordio.md @@ -1,3 +1,22 @@ + + ## Create a Dataset Using RecordIO RecordIO implements a file format for a sequence of records. We recommend storing images as records and packing them together. The benefits include: diff --git a/docs/faq/s3_integration.md b/docs/faq/s3_integration.md index 024356706339..18bd38df71df 100644 --- a/docs/faq/s3_integration.md +++ b/docs/faq/s3_integration.md @@ -1,3 +1,22 @@ + + # Use data from S3 for training AWS S3 is a cloud-based object storage service that allows storage and retrieval of large amounts of data at a very low cost. This makes it an attractive option to store large training datasets. MXNet is deeply integrated with S3 for this purpose. diff --git a/docs/faq/security.md b/docs/faq/security.md index 0615acda3435..05153e20aaf1 100644 --- a/docs/faq/security.md +++ b/docs/faq/security.md @@ -1,3 +1,22 @@ + + # MXNet Security Best Practices MXNet framework has no built-in security protections. It assumes that the MXNet entities involved in model training and inferencing (hosting) are fully trusted. It also assumes that their communications cannot be eavesdropped or tampered with. MXNet consumers shall ensure that the above assumptions are met. diff --git a/docs/faq/smart_device.md b/docs/faq/smart_device.md index 2584b4c36caf..1c4a8919d981 100644 --- a/docs/faq/smart_device.md +++ b/docs/faq/smart_device.md @@ -1,3 +1,22 @@ + + # Deep Learning in a Single File for Smart Devices Deep learning (DL) systems are complex and often depend on a number of libraries. diff --git a/docs/faq/visualize_graph.md b/docs/faq/visualize_graph.md index 06010213242c..4623346a5a81 100644 --- a/docs/faq/visualize_graph.md +++ b/docs/faq/visualize_graph.md @@ -1,3 +1,22 @@ + + # How to visualize Neural Networks as computation graph Here, we'll demonstrate how to use ```mx.viz.plot_network``` diff --git a/docs/faq/why_mxnet.md b/docs/faq/why_mxnet.md old mode 100755 new mode 100644 index ed8cef143070..00f21043cc41 --- a/docs/faq/why_mxnet.md +++ b/docs/faq/why_mxnet.md @@ -1,3 +1,22 @@ + + # Why MXNet? Probably, if you've stumbled upon this page, you've heard of _deep learning_. diff --git a/docs/gluon/index.md b/docs/gluon/index.md index 96e8e36dbf20..b136efbe0ae5 100644 --- a/docs/gluon/index.md +++ b/docs/gluon/index.md @@ -1,3 +1,22 @@ + + # About Gluon ![gluon logo](https://github.com/dmlc/web-data/blob/master/mxnet/image/image-gluon-logo.png?raw=true) diff --git a/docs/index.md b/docs/index.md index ab6a95dc0ddd..1ecdbde909d5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,3 +1,22 @@ + + # MXNet ```eval_rst diff --git a/docs/install/amazonlinux_setup.md b/docs/install/amazonlinux_setup.md index 42a4fcb0eb89..baac53461ffd 100644 --- a/docs/install/amazonlinux_setup.md +++ b/docs/install/amazonlinux_setup.md @@ -1,3 +1,22 @@ + + diff --git a/docs/install/build_from_source.md b/docs/install/build_from_source.md index e807fb44b599..0effd3b30979 100644 --- a/docs/install/build_from_source.md +++ b/docs/install/build_from_source.md @@ -1,3 +1,22 @@ + + # Build MXNet from Source This document explains how to build MXNet from source code. diff --git a/docs/install/c_plus_plus.md b/docs/install/c_plus_plus.md index 6ad67e2803db..f95bcc645746 100644 --- a/docs/install/c_plus_plus.md +++ b/docs/install/c_plus_plus.md @@ -1,3 +1,22 @@ + + ## Build the C++ package The C++ package has the same prerequisites as the MXNet library. diff --git a/docs/install/centos_setup.md b/docs/install/centos_setup.md index e5efe42a61dd..a42c337c0f91 100644 --- a/docs/install/centos_setup.md +++ b/docs/install/centos_setup.md @@ -1,3 +1,22 @@ + + # Installing MXNet on CentOS and other non-Ubuntu Linux systems Step 1. Install build tools and git on `CentOS >= 7` and `Fedora >= 19`: diff --git a/docs/install/download.md b/docs/install/download.md index d7b9440e0a98..998336731d5f 100644 --- a/docs/install/download.md +++ b/docs/install/download.md @@ -1,3 +1,22 @@ + + # Source Download These source archives are generated from tagged releases. Updates and patches will not have been applied. For any updates refer to the corresponding branches in the [GitHub repository](https://github.com/apache/incubator-mxnet). Choose your flavor of download from the following links: diff --git a/docs/install/index.md b/docs/install/index.md index ad3d083a7d02..c4da719e72d9 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -1,3 +1,22 @@ + + # Installing MXNet ```eval_rst diff --git a/docs/install/java_setup.md b/docs/install/java_setup.md index bd20c9596013..39765730353a 100644 --- a/docs/install/java_setup.md +++ b/docs/install/java_setup.md @@ -1,3 +1,22 @@ + + # Setup the MXNet Package for Java The following instructions are provided for macOS and Ubuntu. Windows is not yet available. diff --git a/docs/install/osx_setup.md b/docs/install/osx_setup.md index 7d90d3d456f6..e39c006b86ee 100644 --- a/docs/install/osx_setup.md +++ b/docs/install/osx_setup.md @@ -1,3 +1,22 @@ + + # Installing MXNet from source on OS X (Mac) **NOTE:** For prebuild MXNet with Python installation, please refer to the [new install guide](http://mxnet.io/install/index.html). diff --git a/docs/install/raspbian_setup.md b/docs/install/raspbian_setup.md index 42a4fcb0eb89..baac53461ffd 100644 --- a/docs/install/raspbian_setup.md +++ b/docs/install/raspbian_setup.md @@ -1,3 +1,22 @@ + + diff --git a/docs/install/scala_setup.md b/docs/install/scala_setup.md index 9ee9ceac3a3c..a09fcc949e8f 100644 --- a/docs/install/scala_setup.md +++ b/docs/install/scala_setup.md @@ -1,3 +1,22 @@ + + # Setup the MXNet Package for Scala The following instructions are provided for macOS and Ubuntu. Windows is not yet available. diff --git a/docs/install/tx2_setup.md b/docs/install/tx2_setup.md index 42a4fcb0eb89..baac53461ffd 100644 --- a/docs/install/tx2_setup.md +++ b/docs/install/tx2_setup.md @@ -1,3 +1,22 @@ + + diff --git a/docs/install/ubuntu_setup.md b/docs/install/ubuntu_setup.md index bf964182b50a..726888f377cb 100644 --- a/docs/install/ubuntu_setup.md +++ b/docs/install/ubuntu_setup.md @@ -1,3 +1,22 @@ + + # Installing MXNet on Ubuntu The following installation instructions are for installing MXNet on computers running **Ubuntu 16.04**. Support for later versions of Ubuntu is [not yet available](#contributions). diff --git a/docs/install/validate_mxnet.md b/docs/install/validate_mxnet.md index dfe8d063f602..1dd4df89b149 100644 --- a/docs/install/validate_mxnet.md +++ b/docs/install/validate_mxnet.md @@ -1,3 +1,22 @@ + + # Validate Your MXNet Installation - [Python](#python) diff --git a/docs/install/windows_setup.md b/docs/install/windows_setup.md old mode 100755 new mode 100644 index b34936140aea..383bca0b9778 --- a/docs/install/windows_setup.md +++ b/docs/install/windows_setup.md @@ -1,3 +1,22 @@ + + # Installing MXNet on Windows The following describes how to install with pip for computers with CPUs, Intel CPUs, and NVIDIA GPUs. Further along in the document you can learn how to build MXNet from source on Windows, or how to install packages that support different language APIs to MXNet. diff --git a/docs/model_zoo/index.md b/docs/model_zoo/index.md index 034d360b985a..0a65a8d68ff8 100644 --- a/docs/model_zoo/index.md +++ b/docs/model_zoo/index.md @@ -1,3 +1,22 @@ + + # MXNet Model Zoo MXNet features fast implementations of many state-of-the-art models reported in the academic literature. This Model Zoo is an diff --git a/docs/settings.ini b/docs/settings.ini index e16177604e8d..ec64dd784cdb 100644 --- a/docs/settings.ini +++ b/docs/settings.ini @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [mxnet] build_mxnet = 0 diff --git a/docs/tutorials/basic/data.md b/docs/tutorials/basic/data.md index 4a682e83f9fe..e62ea4fa9737 100644 --- a/docs/tutorials/basic/data.md +++ b/docs/tutorials/basic/data.md @@ -1,3 +1,22 @@ + + # Iterators - Loading data In this tutorial, we focus on how to feed data into a training or inference program. Most training and inference modules in MXNet accept data iterators, diff --git a/docs/tutorials/basic/index.md b/docs/tutorials/basic/index.md index 87d72894424f..aab7731342f7 100644 --- a/docs/tutorials/basic/index.md +++ b/docs/tutorials/basic/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/basic/module.md b/docs/tutorials/basic/module.md index f7a4d6e25de7..67f5d8aecd1f 100644 --- a/docs/tutorials/basic/module.md +++ b/docs/tutorials/basic/module.md @@ -1,3 +1,22 @@ + + # Module - Neural network training and inference diff --git a/docs/tutorials/basic/ndarray.md b/docs/tutorials/basic/ndarray.md index 2c171f2627e8..00cac64dd6ba 100644 --- a/docs/tutorials/basic/ndarray.md +++ b/docs/tutorials/basic/ndarray.md @@ -1,3 +1,22 @@ + + # NDArray - Imperative tensor operations on CPU/GPU In _MXNet_, `NDArray` is the core data structure for all mathematical diff --git a/docs/tutorials/basic/ndarray_indexing.md b/docs/tutorials/basic/ndarray_indexing.md index 35dd8c17f675..e06e5cb4482d 100644 --- a/docs/tutorials/basic/ndarray_indexing.md +++ b/docs/tutorials/basic/ndarray_indexing.md @@ -1,3 +1,22 @@ + + # NDArray Indexing - Array indexing features diff --git a/docs/tutorials/basic/reshape_transpose.md b/docs/tutorials/basic/reshape_transpose.md index 999b22ca2f7e..7407920b0aea 100644 --- a/docs/tutorials/basic/reshape_transpose.md +++ b/docs/tutorials/basic/reshape_transpose.md @@ -1,3 +1,22 @@ + + ## Difference between reshape and transpose operators What does it mean if MXNet gives you an error like the this? diff --git a/docs/tutorials/basic/symbol.md b/docs/tutorials/basic/symbol.md index 5e1e3cd8c62f..5642bab74d64 100644 --- a/docs/tutorials/basic/symbol.md +++ b/docs/tutorials/basic/symbol.md @@ -1,3 +1,22 @@ + + # Symbol - Neural network graphs In a [previous tutorial](http://mxnet.io/tutorials/basic/ndarray.html), we introduced `NDArray`, diff --git a/docs/tutorials/c++/basics.md b/docs/tutorials/c++/basics.md index aa73a7363b1c..a960b1817635 100644 --- a/docs/tutorials/c++/basics.md +++ b/docs/tutorials/c++/basics.md @@ -1,3 +1,22 @@ + + Basics ====== diff --git a/docs/tutorials/c++/index.md b/docs/tutorials/c++/index.md index 87d72894424f..aab7731342f7 100644 --- a/docs/tutorials/c++/index.md +++ b/docs/tutorials/c++/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md b/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md index ab55a0e787f1..c408b8a06f52 100644 --- a/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md +++ b/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md @@ -1,3 +1,22 @@ + + # MXNet C++ API inference tutorial ## Overview diff --git a/docs/tutorials/c++/subgraphAPI.md b/docs/tutorials/c++/subgraphAPI.md index 0ae4341b287c..a830bcbcf422 100644 --- a/docs/tutorials/c++/subgraphAPI.md +++ b/docs/tutorials/c++/subgraphAPI.md @@ -1,3 +1,22 @@ + + ## Subgraph API The subgraph API has been proposed and implemented as the default mechanism for integrating backend libraries to MXNet. The subgraph API is a very flexible interface. Although it was proposed as an integration mechanism, it has been used as a tool for manipulating NNVM graphs for graph-level optimizations, such as operator fusion. diff --git a/docs/tutorials/control_flow/ControlFlowTutorial.md b/docs/tutorials/control_flow/ControlFlowTutorial.md index 4b6a23136b5d..173027b13bcf 100644 --- a/docs/tutorials/control_flow/ControlFlowTutorial.md +++ b/docs/tutorials/control_flow/ControlFlowTutorial.md @@ -1,3 +1,22 @@ + + # Hybridize Gluon models with control flows. MXNet currently provides three control flow operators: `cond`, `foreach` and `while_loop`. Like other MXNet operators, they all have a version for NDArray and a version for Symbol. These two versions have exactly the same semantics. We can take advantage of this and use them in Gluon to hybridize models. diff --git a/docs/tutorials/control_flow/index.md b/docs/tutorials/control_flow/index.md index 87d72894424f..aab7731342f7 100644 --- a/docs/tutorials/control_flow/index.md +++ b/docs/tutorials/control_flow/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/embedded/index.md b/docs/tutorials/embedded/index.md index 87d72894424f..aab7731342f7 100644 --- a/docs/tutorials/embedded/index.md +++ b/docs/tutorials/embedded/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/embedded/wine_detector.md b/docs/tutorials/embedded/wine_detector.md index 65e6fbaa4d91..b59a52bd8649 100644 --- a/docs/tutorials/embedded/wine_detector.md +++ b/docs/tutorials/embedded/wine_detector.md @@ -1,3 +1,22 @@ + + # Real-time Object Detection with MXNet On The Raspberry Pi This tutorial shows developers who work with the Raspberry Pi or similar embedded ARM-based devices how to compile MXNet for those devices and run a pretrained deep network model. It also shows how to use AWS IoT to manage and monitor MXNet models running on your devices. diff --git a/docs/tutorials/gluon/autograd.md b/docs/tutorials/gluon/autograd.md index 4b296dd2dd5b..495db34de825 100644 --- a/docs/tutorials/gluon/autograd.md +++ b/docs/tutorials/gluon/autograd.md @@ -1,3 +1,22 @@ + + # Automatic differentiation MXNet supports automatic differentiation with the `autograd` package. diff --git a/docs/tutorials/gluon/custom_layer.md b/docs/tutorials/gluon/custom_layer.md index 97bdf05aff58..a45622d175cc 100644 --- a/docs/tutorials/gluon/custom_layer.md +++ b/docs/tutorials/gluon/custom_layer.md @@ -1,3 +1,22 @@ + + # How to write a custom layer in Apache MxNet Gluon API diff --git a/docs/tutorials/gluon/customop.md b/docs/tutorials/gluon/customop.md index df10788d6788..5552093484ba 100644 --- a/docs/tutorials/gluon/customop.md +++ b/docs/tutorials/gluon/customop.md @@ -1,3 +1,22 @@ + + # Creating custom operators with numpy diff --git a/docs/tutorials/gluon/data_augmentation.md b/docs/tutorials/gluon/data_augmentation.md index df7462c33044..ce631cea5fbf 100644 --- a/docs/tutorials/gluon/data_augmentation.md +++ b/docs/tutorials/gluon/data_augmentation.md @@ -1,3 +1,22 @@ + + # Methods of applying data augmentation (Gluon API) Data Augmentation is a regularization technique that's used to avoid overfitting when training Machine Learning models. Although the technique can be applied in a variety of domains, it's very common in Computer Vision. Adjustments are made to the original images in the training dataset before being used in training. Some example adjustments include translating, cropping, scaling, rotating, changing brightness and contrast. We do this to reduce the dependence of the model on spurious characteristics; e.g. training data may only contain faces that fill 1/4 of the image, so the model trained without data augmentation might unhelpfully learn that faces can only be of this size. diff --git a/docs/tutorials/gluon/datasets.md b/docs/tutorials/gluon/datasets.md index 0b0038def633..94b27bbbd331 100644 --- a/docs/tutorials/gluon/datasets.md +++ b/docs/tutorials/gluon/datasets.md @@ -1,3 +1,22 @@ + + # Gluon `Dataset`s and `DataLoader` diff --git a/docs/tutorials/gluon/gluon.md b/docs/tutorials/gluon/gluon.md index 518e99905c04..6cf27ff603d2 100644 --- a/docs/tutorials/gluon/gluon.md +++ b/docs/tutorials/gluon/gluon.md @@ -1,3 +1,22 @@ + + # Gluon - Neural network building blocks Gluon package is a high-level interface for MXNet designed to be easy to use while diff --git a/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md b/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md index a3a6aab3593c..0394c84d1e02 100644 --- a/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md +++ b/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md @@ -1,3 +1,22 @@ + + # Gluon: from experiment to deployment, an end to end tutorial diff --git a/docs/tutorials/gluon/gotchas_numpy_in_mxnet.md b/docs/tutorials/gluon/gotchas_numpy_in_mxnet.md index c82c63edbc2b..4b2fa95ce885 100644 --- a/docs/tutorials/gluon/gotchas_numpy_in_mxnet.md +++ b/docs/tutorials/gluon/gotchas_numpy_in_mxnet.md @@ -1,3 +1,22 @@ + + # Gotchas using NumPy in Apache MXNet diff --git a/docs/tutorials/gluon/hybrid.md b/docs/tutorials/gluon/hybrid.md index 17e9e1b20d74..c79f79692637 100644 --- a/docs/tutorials/gluon/hybrid.md +++ b/docs/tutorials/gluon/hybrid.md @@ -1,3 +1,22 @@ + + # Hybrid - Faster training and easy deployment *Related Content:* diff --git a/docs/tutorials/gluon/index.md b/docs/tutorials/gluon/index.md index 87d72894424f..aab7731342f7 100644 --- a/docs/tutorials/gluon/index.md +++ b/docs/tutorials/gluon/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/gluon/info_gan.md b/docs/tutorials/gluon/info_gan.md index 8b2668ab22e6..2b8cdfe3fc4e 100644 --- a/docs/tutorials/gluon/info_gan.md +++ b/docs/tutorials/gluon/info_gan.md @@ -1,3 +1,22 @@ + + # Image similarity search with InfoGAN diff --git a/docs/tutorials/gluon/learning_rate_finder.md b/docs/tutorials/gluon/learning_rate_finder.md index b571a53f674c..b4627f25bc39 100644 --- a/docs/tutorials/gluon/learning_rate_finder.md +++ b/docs/tutorials/gluon/learning_rate_finder.md @@ -1,3 +1,22 @@ + + # Learning Rate Finder diff --git a/docs/tutorials/gluon/learning_rate_schedules.md b/docs/tutorials/gluon/learning_rate_schedules.md index 88b109e7f33e..3416ab44f992 100644 --- a/docs/tutorials/gluon/learning_rate_schedules.md +++ b/docs/tutorials/gluon/learning_rate_schedules.md @@ -1,3 +1,22 @@ + + # Learning Rate Schedules diff --git a/docs/tutorials/gluon/learning_rate_schedules_advanced.md b/docs/tutorials/gluon/learning_rate_schedules_advanced.md index bdaf0a9ba38d..0d933997a2bb 100644 --- a/docs/tutorials/gluon/learning_rate_schedules_advanced.md +++ b/docs/tutorials/gluon/learning_rate_schedules_advanced.md @@ -1,3 +1,22 @@ + + # Advanced Learning Rate Schedules diff --git a/docs/tutorials/gluon/logistic_regression_explained.md b/docs/tutorials/gluon/logistic_regression_explained.md index 577a91413b33..93a93ccc9c32 100644 --- a/docs/tutorials/gluon/logistic_regression_explained.md +++ b/docs/tutorials/gluon/logistic_regression_explained.md @@ -1,3 +1,22 @@ + + # Logistic regression using Gluon API explained diff --git a/docs/tutorials/gluon/mnist.md b/docs/tutorials/gluon/mnist.md index 35fb40521f62..aa1efccfb653 100644 --- a/docs/tutorials/gluon/mnist.md +++ b/docs/tutorials/gluon/mnist.md @@ -1,3 +1,22 @@ + + # Hand-written Digit Recognition In this tutorial, we'll give you a step-by-step walkthrough of building a hand-written digit classifier using the [MNIST](https://en.wikipedia.org/wiki/MNIST_database) dataset. diff --git a/docs/tutorials/gluon/naming.md b/docs/tutorials/gluon/naming.md index 3606a03dcbd2..16383d309fc5 100644 --- a/docs/tutorials/gluon/naming.md +++ b/docs/tutorials/gluon/naming.md @@ -1,3 +1,22 @@ + + # Naming of Gluon Parameter and Blocks diff --git a/docs/tutorials/gluon/ndarray.md b/docs/tutorials/gluon/ndarray.md index 7cf08a88cbf3..f4bae9494454 100644 --- a/docs/tutorials/gluon/ndarray.md +++ b/docs/tutorials/gluon/ndarray.md @@ -1,3 +1,22 @@ + + # NDArray - Scientific computing on CPU and GPU NDArray is a tensor data structure similar to numpy's multi-dimensional array. diff --git a/docs/tutorials/gluon/pretrained_models.md b/docs/tutorials/gluon/pretrained_models.md index 0de5fdd0b44f..796fe05e731e 100644 --- a/docs/tutorials/gluon/pretrained_models.md +++ b/docs/tutorials/gluon/pretrained_models.md @@ -1,3 +1,22 @@ + + # Using pre-trained models in MXNet diff --git a/docs/tutorials/gluon/save_load_params.md b/docs/tutorials/gluon/save_load_params.md index ebc8103e7b45..feaf5eaf8547 100644 --- a/docs/tutorials/gluon/save_load_params.md +++ b/docs/tutorials/gluon/save_load_params.md @@ -1,3 +1,22 @@ + + # Saving and Loading Gluon Models Training large models take a lot of time and it is a good idea to save the trained models to files to avoid training them again and again. There are a number of reasons to do this. For example, you might want to do inference on a machine that is different from the one where the model was trained. Sometimes model's performance on validation set decreases towards the end of the training because of overfitting. If you saved your model parameters after every epoch, at the end you can decide to use the model that performs best on the validation set. Another reason would be to train your model using one language (like Python that has a lot of tools for training) and run inference using a different language (like Scala probably because your application is built on Scala). diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index cad9099fcc71..562c66170a01 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/java/index.md b/docs/tutorials/java/index.md index 87d72894424f..aab7731342f7 100644 --- a/docs/tutorials/java/index.md +++ b/docs/tutorials/java/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/java/mxnet_java_on_intellij.md b/docs/tutorials/java/mxnet_java_on_intellij.md index ef2c009f66e8..b36ee3cb0ebc 100644 --- a/docs/tutorials/java/mxnet_java_on_intellij.md +++ b/docs/tutorials/java/mxnet_java_on_intellij.md @@ -1,3 +1,22 @@ + + # Run MXNet Java Examples Using the IntelliJ IDE (macOS) This tutorial guides you through setting up a simple Java project in IntelliJ IDE on macOS and demonstrates usage of the MXNet Java APIs. diff --git a/docs/tutorials/java/ssd_inference.md b/docs/tutorials/java/ssd_inference.md index 3a20329f9a91..b61fe97036ef 100644 --- a/docs/tutorials/java/ssd_inference.md +++ b/docs/tutorials/java/ssd_inference.md @@ -1,3 +1,22 @@ + + # Multi Object Detection using pre-trained SSD Model via Java Inference APIs This tutorial shows how to use MXNet Java Inference APIs to run inference on a pre-trained Single Shot Detector (SSD) Model. diff --git a/docs/tutorials/nlp/cnn.md b/docs/tutorials/nlp/cnn.md index b3d7d0d38941..128f5caaae26 100644 --- a/docs/tutorials/nlp/cnn.md +++ b/docs/tutorials/nlp/cnn.md @@ -1,3 +1,22 @@ + + # Text Classification Using a Convolutional Neural Network on MXNet This tutorial is based of Yoon Kim's [paper](https://arxiv.org/abs/1408.5882) on using convolutional neural networks for sentence sentiment classification. The tutorial has been tested on MXNet 1.0 running under Python 2.7 and Python 3.6. diff --git a/docs/tutorials/nlp/index.md b/docs/tutorials/nlp/index.md index 87d72894424f..aab7731342f7 100644 --- a/docs/tutorials/nlp/index.md +++ b/docs/tutorials/nlp/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/onnx/export_mxnet_to_onnx.md b/docs/tutorials/onnx/export_mxnet_to_onnx.md index 3f925c7b5b84..6190b9242a74 100644 --- a/docs/tutorials/onnx/export_mxnet_to_onnx.md +++ b/docs/tutorials/onnx/export_mxnet_to_onnx.md @@ -1,3 +1,22 @@ + + # Exporting MXNet model to ONNX format diff --git a/docs/tutorials/onnx/fine_tuning_gluon.md b/docs/tutorials/onnx/fine_tuning_gluon.md index 750a6757272f..70c1469afa94 100644 --- a/docs/tutorials/onnx/fine_tuning_gluon.md +++ b/docs/tutorials/onnx/fine_tuning_gluon.md @@ -1,3 +1,22 @@ + + # Fine-tuning an ONNX model with MXNet/Gluon diff --git a/docs/tutorials/onnx/index.md b/docs/tutorials/onnx/index.md index 87d72894424f..aab7731342f7 100644 --- a/docs/tutorials/onnx/index.md +++ b/docs/tutorials/onnx/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/onnx/inference_on_onnx_model.md b/docs/tutorials/onnx/inference_on_onnx_model.md index b2522ad0c1f1..f500956f72c6 100644 --- a/docs/tutorials/onnx/inference_on_onnx_model.md +++ b/docs/tutorials/onnx/inference_on_onnx_model.md @@ -1,3 +1,22 @@ + + # Running inference on MXNet/Gluon from an ONNX model diff --git a/docs/tutorials/onnx/super_resolution.md b/docs/tutorials/onnx/super_resolution.md index 36c06b743c8e..d9825090e8ee 100644 --- a/docs/tutorials/onnx/super_resolution.md +++ b/docs/tutorials/onnx/super_resolution.md @@ -1,3 +1,22 @@ + + # Importing an ONNX model into MXNet In this tutorial we will: diff --git a/docs/tutorials/python/data_augmentation.md b/docs/tutorials/python/data_augmentation.md index e4dbbb672997..1034f621ba6f 100644 --- a/docs/tutorials/python/data_augmentation.md +++ b/docs/tutorials/python/data_augmentation.md @@ -1,3 +1,22 @@ + + # Methods of applying data augmentation (Module API) Data Augmentation is a regularization technique that's used to avoid overfitting when training Machine Learning models. Although the technique can be applied in a variety of domains, it's very common in Computer Vision. Adjustments are made to the original images in the training dataset before being used in training. Some example adjustments include translating, cropping, scaling, rotating, changing brightness and contrast. We do this to reduce the dependence of the model on spurious characteristics; e.g. training data may only contain faces that fill 1/4 of the image, so the model trained without data augmentation might unhelpfully learn that faces can only be of this size. diff --git a/docs/tutorials/python/data_augmentation_with_masks.md b/docs/tutorials/python/data_augmentation_with_masks.md index ac587ac2f5e2..080708caedb1 100644 --- a/docs/tutorials/python/data_augmentation_with_masks.md +++ b/docs/tutorials/python/data_augmentation_with_masks.md @@ -1,3 +1,22 @@ + + # Data Augmentation with Masks diff --git a/docs/tutorials/python/index.md b/docs/tutorials/python/index.md index 87d72894424f..aab7731342f7 100644 --- a/docs/tutorials/python/index.md +++ b/docs/tutorials/python/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/python/kvstore.md b/docs/tutorials/python/kvstore.md index 3e6bbf12c393..593c240ad17c 100644 --- a/docs/tutorials/python/kvstore.md +++ b/docs/tutorials/python/kvstore.md @@ -1,3 +1,22 @@ + + # Distributed Key-Value Store KVStore is a place for data sharing. Think of it as a single object shared diff --git a/docs/tutorials/python/linear-regression.md b/docs/tutorials/python/linear-regression.md index fd336ad2aed5..77ca7f9a7ceb 100644 --- a/docs/tutorials/python/linear-regression.md +++ b/docs/tutorials/python/linear-regression.md @@ -1,3 +1,22 @@ + + # Linear Regression In this tutorial we'll walk through how one can implement *linear regression* using MXNet APIs. diff --git a/docs/tutorials/python/matrix_factorization.md b/docs/tutorials/python/matrix_factorization.md index 154fa4b3e127..62674e7ad646 100644 --- a/docs/tutorials/python/matrix_factorization.md +++ b/docs/tutorials/python/matrix_factorization.md @@ -1,3 +1,22 @@ + + # Matrix Factorization In a recommendation system, there is a group of users and a set of items. Given diff --git a/docs/tutorials/python/mnist.md b/docs/tutorials/python/mnist.md index df949d487b63..a8eee40e095e 100644 --- a/docs/tutorials/python/mnist.md +++ b/docs/tutorials/python/mnist.md @@ -1,3 +1,22 @@ + + # Handwritten Digit Recognition In this tutorial, we'll give you a step by step walk-through of how to build a hand-written digit classifier using the [MNIST](https://en.wikipedia.org/wiki/MNIST_database) dataset. For someone new to deep learning, this exercise is arguably the "Hello World" equivalent. diff --git a/docs/tutorials/python/predict_image.md b/docs/tutorials/python/predict_image.md index 8be98d991366..e02db165d7f6 100644 --- a/docs/tutorials/python/predict_image.md +++ b/docs/tutorials/python/predict_image.md @@ -1,3 +1,22 @@ + + # Predict with pre-trained models This tutorial explains how to recognize objects in an image with a pre-trained model, and how to perform feature extraction. diff --git a/docs/tutorials/python/profiler.md b/docs/tutorials/python/profiler.md index 7dcda10f11b8..1d6c0b2fc7f6 100644 --- a/docs/tutorials/python/profiler.md +++ b/docs/tutorials/python/profiler.md @@ -1,3 +1,22 @@ + + # Profiling MXNet Models It is often helpful to understand what operations take how much time while running a model. This helps optimize the model to run faster. In this tutorial, we will learn how to profile MXNet models to measure their running time and memory consumption using the MXNet profiler. diff --git a/docs/tutorials/python/types_of_data_augmentation.md b/docs/tutorials/python/types_of_data_augmentation.md index 4308932bf483..344f3cd4d34a 100644 --- a/docs/tutorials/python/types_of_data_augmentation.md +++ b/docs/tutorials/python/types_of_data_augmentation.md @@ -1,3 +1,22 @@ + + # Types of Data Augmentation diff --git a/docs/tutorials/r/CallbackFunction.md b/docs/tutorials/r/CallbackFunction.md index 103352dd2907..f3156cd969e8 100644 --- a/docs/tutorials/r/CallbackFunction.md +++ b/docs/tutorials/r/CallbackFunction.md @@ -1,3 +1,22 @@ + + Callback Function ====================================== diff --git a/docs/tutorials/r/CustomIterator.md b/docs/tutorials/r/CustomIterator.md index 1ad634bcd669..62a618a51590 100644 --- a/docs/tutorials/r/CustomIterator.md +++ b/docs/tutorials/r/CustomIterator.md @@ -1,3 +1,22 @@ + + Custom Iterator Tutorial ====================================== diff --git a/docs/tutorials/r/CustomLossFunction.md b/docs/tutorials/r/CustomLossFunction.md index afb99518894c..87ac40dafc63 100644 --- a/docs/tutorials/r/CustomLossFunction.md +++ b/docs/tutorials/r/CustomLossFunction.md @@ -1,3 +1,22 @@ + + Customized loss function ====================================== diff --git a/docs/tutorials/r/MultidimLstm.md b/docs/tutorials/r/MultidimLstm.md index 8692086d180b..49a07361c9d4 100644 --- a/docs/tutorials/r/MultidimLstm.md +++ b/docs/tutorials/r/MultidimLstm.md @@ -1,3 +1,22 @@ + + LSTM time series example ============================================= diff --git a/docs/tutorials/r/charRnnModel.md b/docs/tutorials/r/charRnnModel.md index cb21e77559b5..f0d1f9e4d1fb 100644 --- a/docs/tutorials/r/charRnnModel.md +++ b/docs/tutorials/r/charRnnModel.md @@ -1,3 +1,22 @@ + + # Character-level Language Model using RNN diff --git a/docs/tutorials/r/classifyRealImageWithPretrainedModel.md b/docs/tutorials/r/classifyRealImageWithPretrainedModel.md index b2f2035426ec..8a7daf87e4f1 100644 --- a/docs/tutorials/r/classifyRealImageWithPretrainedModel.md +++ b/docs/tutorials/r/classifyRealImageWithPretrainedModel.md @@ -1,3 +1,22 @@ + + Classify Images with a PreTrained Model ================================================= MXNet is a flexible and efficient deep learning framework. One of the interesting things that a deep learning diff --git a/docs/tutorials/r/fiveMinutesNeuralNetwork.md b/docs/tutorials/r/fiveMinutesNeuralNetwork.md index 6d79cd288d2c..56688a65338e 100644 --- a/docs/tutorials/r/fiveMinutesNeuralNetwork.md +++ b/docs/tutorials/r/fiveMinutesNeuralNetwork.md @@ -1,3 +1,22 @@ + + Develop a Neural Network with MXNet in Five Minutes ============================================= diff --git a/docs/tutorials/r/index.md b/docs/tutorials/r/index.md index fbc8911f2a6d..6ab039561886 100644 --- a/docs/tutorials/r/index.md +++ b/docs/tutorials/r/index.md @@ -1,3 +1,22 @@ + + # R Tutorials These tutorials introduce a few fundamental concepts in deep learning and how to implement them in R using _MXNet_. diff --git a/docs/tutorials/r/mnistCompetition.md b/docs/tutorials/r/mnistCompetition.md index ed3c2827011d..95efbfd6d7af 100644 --- a/docs/tutorials/r/mnistCompetition.md +++ b/docs/tutorials/r/mnistCompetition.md @@ -1,3 +1,22 @@ + + Handwritten Digits Classification Competition ============================================= diff --git a/docs/tutorials/r/ndarray.md b/docs/tutorials/r/ndarray.md index cb7639a8a44d..30f5d887a3b6 100644 --- a/docs/tutorials/r/ndarray.md +++ b/docs/tutorials/r/ndarray.md @@ -1,3 +1,22 @@ + + # NDArray: Vectorized Tensor Computations on CPUs and GPUs `NDArray` is the basic vectorized operation unit in MXNet for matrix and tensor computations. diff --git a/docs/tutorials/r/symbol.md b/docs/tutorials/r/symbol.md index 4a87643b9f50..0ab560856bb4 100644 --- a/docs/tutorials/r/symbol.md +++ b/docs/tutorials/r/symbol.md @@ -1,3 +1,22 @@ + + # Symbol and Automatic Differentiation The computational unit `NDArray` requires a way to construct neural networks. MXNet provides a symbolic interface, named Symbol, to do this. Symbol combines both flexibility and efficiency. diff --git a/docs/tutorials/scala/char_lstm.md b/docs/tutorials/scala/char_lstm.md index 4d6a5aee921e..fafd5d5a0bd9 100644 --- a/docs/tutorials/scala/char_lstm.md +++ b/docs/tutorials/scala/char_lstm.md @@ -1,3 +1,22 @@ + + # Developing a Character-level Language model This tutorial shows how to train a character-level language model with a multilayer recurrent neural network (RNN) using Scala. This model takes one text file as input and trains an RNN that learns to predict the next character in the sequence. In this tutorial, you train a multilayer LSTM (Long Short-Term Memory) network that generates relevant text using Barack Obama's speech patterns. diff --git a/docs/tutorials/scala/index.md b/docs/tutorials/scala/index.md index f14337f90f08..55e41e428d38 100644 --- a/docs/tutorials/scala/index.md +++ b/docs/tutorials/scala/index.md @@ -1,3 +1,22 @@ + + # MXNet-Scala Tutorials ## Installation & Setup diff --git a/docs/tutorials/scala/mnist.md b/docs/tutorials/scala/mnist.md index 79f2129ef0ef..1f05fb464fb0 100644 --- a/docs/tutorials/scala/mnist.md +++ b/docs/tutorials/scala/mnist.md @@ -1,3 +1,22 @@ + + # Handwritten Digit Recognition This Scala tutorial guides you through a classic computer vision application: identifying hand written digits. diff --git a/docs/tutorials/scala/mxnet_scala_on_intellij.md b/docs/tutorials/scala/mxnet_scala_on_intellij.md index 769a6b4fe506..cae70567de4d 100644 --- a/docs/tutorials/scala/mxnet_scala_on_intellij.md +++ b/docs/tutorials/scala/mxnet_scala_on_intellij.md @@ -1,3 +1,22 @@ + + # Run MXNet Scala Examples Using the IntelliJ IDE (macOS) This tutorial guides you through setting up a Scala project in the IntelliJ IDE on macOS, and shows how to use the MXNet package from your application. diff --git a/docs/tutorials/sparse/csr.md b/docs/tutorials/sparse/csr.md index 0aede1ab4313..f1718687cfb0 100644 --- a/docs/tutorials/sparse/csr.md +++ b/docs/tutorials/sparse/csr.md @@ -1,3 +1,22 @@ + + # CSRNDArray - NDArray in Compressed Sparse Row Storage Format diff --git a/docs/tutorials/sparse/index.md b/docs/tutorials/sparse/index.md index 87d72894424f..aab7731342f7 100644 --- a/docs/tutorials/sparse/index.md +++ b/docs/tutorials/sparse/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/sparse/row_sparse.md b/docs/tutorials/sparse/row_sparse.md index 46a5edad075e..2986c4d3ab8f 100644 --- a/docs/tutorials/sparse/row_sparse.md +++ b/docs/tutorials/sparse/row_sparse.md @@ -1,3 +1,22 @@ + + # RowSparseNDArray - NDArray for Sparse Gradient Updates diff --git a/docs/tutorials/sparse/train.md b/docs/tutorials/sparse/train.md index fde4c0e65521..d22fbe9e8f0c 100644 --- a/docs/tutorials/sparse/train.md +++ b/docs/tutorials/sparse/train.md @@ -1,3 +1,22 @@ + + # Train a Linear Regression Model with Sparse Symbols In previous tutorials, we introduced `CSRNDArray` and `RowSparseNDArray`, diff --git a/docs/tutorials/speech_recognition/ctc.md b/docs/tutorials/speech_recognition/ctc.md index 0b01fb48999c..3f1b8d4ff9f7 100644 --- a/docs/tutorials/speech_recognition/ctc.md +++ b/docs/tutorials/speech_recognition/ctc.md @@ -1,3 +1,22 @@ + + # Connectionist Temporal Classification ```python diff --git a/docs/tutorials/speech_recognition/index.md b/docs/tutorials/speech_recognition/index.md index 87d72894424f..aab7731342f7 100644 --- a/docs/tutorials/speech_recognition/index.md +++ b/docs/tutorials/speech_recognition/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/tensorrt/index.md b/docs/tutorials/tensorrt/index.md index 9515a5b9fd1e..d3dcaeb9db84 100644 --- a/docs/tutorials/tensorrt/index.md +++ b/docs/tutorials/tensorrt/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/tensorrt/inference_with_trt.md b/docs/tutorials/tensorrt/inference_with_trt.md index 489a92ec1014..77f82a7a1039 100644 --- a/docs/tutorials/tensorrt/inference_with_trt.md +++ b/docs/tutorials/tensorrt/inference_with_trt.md @@ -1,3 +1,22 @@ + + # Optimizing Deep Learning Computation Graphs with TensorRT NVIDIA's TensorRT is a deep learning library that has been shown to provide large speedups when used for network inference. MXNet 1.3.0 is shipping with experimental integrated support for TensorRT. This means MXNet users can noew make use of this acceleration library to efficiently run their networks. In this blog post we'll see how to install, enable and run TensorRT with MXNet. We'll also give some insight into what is happening behind the scenes in MXNet to enable TensorRT graph execution. diff --git a/docs/tutorials/unsupervised_learning/gan.md b/docs/tutorials/unsupervised_learning/gan.md index 0efdc5565519..9c50fcfdf15b 100644 --- a/docs/tutorials/unsupervised_learning/gan.md +++ b/docs/tutorials/unsupervised_learning/gan.md @@ -1,3 +1,22 @@ + + # Generative Adversarial Network (GAN) diff --git a/docs/tutorials/unsupervised_learning/index.md b/docs/tutorials/unsupervised_learning/index.md index 87d72894424f..aab7731342f7 100644 --- a/docs/tutorials/unsupervised_learning/index.md +++ b/docs/tutorials/unsupervised_learning/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/vision/cnn_visualization.md b/docs/tutorials/vision/cnn_visualization.md index 5ded6f1587e0..f528d35bfd32 100644 --- a/docs/tutorials/vision/cnn_visualization.md +++ b/docs/tutorials/vision/cnn_visualization.md @@ -1,3 +1,22 @@ + + # Visualizing Decisions of Convolutional Neural Networks Convolutional Neural Networks have made a lot of progress in Computer Vision. Their accuracy is as good as humans in some tasks. However, it remains difficult to explain the predictions of convolutional neural networks, as they lack the interpretability offered by other models such as decision trees. diff --git a/docs/tutorials/vision/index.md b/docs/tutorials/vision/index.md index 87d72894424f..aab7731342f7 100644 --- a/docs/tutorials/vision/index.md +++ b/docs/tutorials/vision/index.md @@ -1,3 +1,22 @@ + + # Tutorials ```eval_rst diff --git a/docs/tutorials/vision/large_scale_classification.md b/docs/tutorials/vision/large_scale_classification.md index aac03e4dd903..5fa87b14821d 100644 --- a/docs/tutorials/vision/large_scale_classification.md +++ b/docs/tutorials/vision/large_scale_classification.md @@ -1,3 +1,22 @@ + + # Large Scale Image Classification Training a neural network with a large number of images presents several challenges. Even with the latest GPUs, it is not possible to train large networks using a large number of images in a reasonable amount of time using a single GPU. This problem can be somewhat mitigated by using multiple GPUs in a single machine. But there is a limit to the number of GPUs that can be attached to one machine (typically 8 or 16). This tutorial explains how to train large networks with terabytes of data using multiple machines each containing multiple GPUs. diff --git a/example/README.md b/example/README.md index dea7e289e6cd..5594e3ed0f3d 100644 --- a/example/README.md +++ b/example/README.md @@ -1,3 +1,22 @@ + + # MXNet Examples This page contains a curated list of awesome MXNet examples, tutorials and blogs. It is inspired by [awesome-php](https://github.com/ziadoz/awesome-php) and [awesome-machine-learning](https://github.com/josephmisiti/awesome-machine-learning). See also [Awesome-MXNet](https://github.com/chinakook/Awesome-MXNet) for a similar list. diff --git a/example/adversary/README.md b/example/adversary/README.md index 5d5b44fb91ba..7f5158c03bbf 100644 --- a/example/adversary/README.md +++ b/example/adversary/README.md @@ -1,3 +1,22 @@ + + # Adversarial examples This demonstrates the concept of "adversarial examples" from [1] showing how to fool a well-trained CNN. diff --git a/example/autoencoder/README.md b/example/autoencoder/README.md index 960636cd7d59..15f4851412af 100644 --- a/example/autoencoder/README.md +++ b/example/autoencoder/README.md @@ -1,3 +1,22 @@ + + # Example of a Convolutional Autoencoder Autoencoder architectures are often used for unsupervised feature learning. This [link](http://ufldl.stanford.edu/tutorial/unsupervised/Autoencoders/) contains an introduction tutorial to autoencoders. This example illustrates a simple autoencoder using a stack of convolutional layers for both the encoder and the decoder. diff --git a/example/autoencoder/variational_autoencoder/README.md b/example/autoencoder/variational_autoencoder/README.md index c6e68d54c4f9..07d099848b46 100644 --- a/example/autoencoder/variational_autoencoder/README.md +++ b/example/autoencoder/variational_autoencoder/README.md @@ -1,3 +1,22 @@ + + Variational Auto Encoder(VAE) ============================= diff --git a/example/bayesian-methods/README.md b/example/bayesian-methods/README.md index fc35b94219d7..56e4b1dd92eb 100644 --- a/example/bayesian-methods/README.md +++ b/example/bayesian-methods/README.md @@ -1,3 +1,22 @@ + + Bayesian Methods ================ diff --git a/example/bi-lstm-sort/README.md b/example/bi-lstm-sort/README.md index f00cc85caa30..e08db18a9bcf 100644 --- a/example/bi-lstm-sort/README.md +++ b/example/bi-lstm-sort/README.md @@ -1,3 +1,22 @@ + + # Bidirectionnal LSTM to sort an array. This is an example of using bidirectionmal lstm to sort an array. Please refer to the notebook. diff --git a/example/caffe/README.md b/example/caffe/README.md index 466305cc9b88..6541d4dacc04 100644 --- a/example/caffe/README.md +++ b/example/caffe/README.md @@ -1,3 +1,22 @@ + + # How to use Caffe operator in MXNet [Caffe](http://caffe.berkeleyvision.org/) has been a well-known and widely-used deep learning framework. Now MXNet has supported calling most caffe operators(layers) and loss functions directly in its symbolic graph! Using one's own customized caffe layer is also effortless. diff --git a/example/capsnet/README.md b/example/capsnet/README.md index 500c7df72515..4c05781a3971 100644 --- a/example/capsnet/README.md +++ b/example/capsnet/README.md @@ -1,3 +1,22 @@ + + **CapsNet-MXNet** ========================================= diff --git a/example/captcha/README.md b/example/captcha/README.md index cc97442f6207..3936178ebd3e 100644 --- a/example/captcha/README.md +++ b/example/captcha/README.md @@ -1,3 +1,22 @@ + + This is the R version of [captcha recognition](http://blog.xlvector.net/2016-05/mxnet-ocr-cnn/) example by xlvector and it can be used as an example of multi-label training. For a captcha below, we consider it as an image with 4 labels and train a CNN over the data set. ![](captcha_example.png) diff --git a/example/cnn_chinese_text_classification/README.md b/example/cnn_chinese_text_classification/README.md index bfb271dd5c45..5bdaecb8c155 100644 --- a/example/cnn_chinese_text_classification/README.md +++ b/example/cnn_chinese_text_classification/README.md @@ -1,3 +1,22 @@ + + Implementing CNN + Highway Network for Chinese Text Classification in MXNet ============ Sentiment classification forked from [incubator-mxnet/cnn_text_classification/](https://github.com/apache/incubator-mxnet/tree/master/example/cnn_text_classification), i've implemented the [Highway Networks](https://arxiv.org/pdf/1505.00387.pdf) architecture.The final train model is CNN + Highway Network structure, and this version can achieve a best dev accuracy of 94.75% with the Chinese corpus. diff --git a/example/cnn_text_classification/README.md b/example/cnn_text_classification/README.md index 2f1991f319ac..c04a22cde103 100644 --- a/example/cnn_text_classification/README.md +++ b/example/cnn_text_classification/README.md @@ -1,3 +1,22 @@ + + Implementing CNN for Text Classification in MXNet ============ It is a slightly simplified implementation of Kim's [Convolutional Neural Networks for Sentence Classification](http://arxiv.org/abs/1408.5882) paper in MXNet. diff --git a/example/ctc/README.md b/example/ctc/README.md index a2f54cffaf86..96373db002d8 100644 --- a/example/ctc/README.md +++ b/example/ctc/README.md @@ -1,3 +1,22 @@ + + # Connectionist Temporal Classification [Connectionist Temporal Classification](https://www.cs.toronto.edu/~graves/icml_2006.pdf) (CTC) is a cost function that is used to train Recurrent Neural Networks (RNNs) to label unsegmented input sequence data in supervised learning. For example in a speech recognition application, using a typical cross-entropy loss the input signal needs to be segmented into words or sub-words. However, using CTC-loss, a single unaligned label sequence per input sequence is sufficient for the network to learn both the alignment and labeling. Baidu's warp-ctc page contains a more detailed [introduction to CTC-loss](https://github.com/baidu-research/warp-ctc#introduction). diff --git a/example/deep-embedded-clustering/README.md b/example/deep-embedded-clustering/README.md index 3972f90bda4a..4e626a6bac81 100644 --- a/example/deep-embedded-clustering/README.md +++ b/example/deep-embedded-clustering/README.md @@ -1,3 +1,22 @@ + + # DEC Implementation This is based on the paper `Unsupervised deep embedding for clustering analysis` by Junyuan Xie, Ross Girshick, and Ali Farhadi diff --git a/example/distributed_training/README.md b/example/distributed_training/README.md index b0b0447725b5..ba09cb91aede 100644 --- a/example/distributed_training/README.md +++ b/example/distributed_training/README.md @@ -1,3 +1,22 @@ + + # Distributed Training using Gluon Deep learning models are usually trained using GPUs because GPUs can do a lot more computations in parallel that CPUs. But even with the modern GPUs, it could take several days to train big models. Training can be done faster by using multiple GPUs like described in [this](https://gluon.mxnet.io/chapter07_distributed-learning/multiple-gpus-gluon.html) tutorial. However only a certain number of GPUs can be attached to one host (typically 8 or 16). To make the training even faster, we can use multiple GPUs attached to multiple hosts. diff --git a/example/dsd/README.md b/example/dsd/README.md index 0ce5cc5d1f0f..bffacce8cabc 100644 --- a/example/dsd/README.md +++ b/example/dsd/README.md @@ -1,3 +1,22 @@ + + DSD Training ============ This folder contains an optimizer class that implements DSD training coupled with SGD. The training diff --git a/example/fcn-xs/README.md b/example/fcn-xs/README.md index 49c57fc08eaf..98d8edab6f93 100644 --- a/example/fcn-xs/README.md +++ b/example/fcn-xs/README.md @@ -1,3 +1,22 @@ + + FCN-xs EXAMPLE -------------- This folder contains an example implementation for Fully Convolutional Networks (FCN) in MXNet. diff --git a/example/gluon/audio/urban_sounds/README.md b/example/gluon/audio/urban_sounds/README.md index c85d29db2e5a..18593272f144 100644 --- a/example/gluon/audio/urban_sounds/README.md +++ b/example/gluon/audio/urban_sounds/README.md @@ -1,3 +1,22 @@ + + # Urban Sounds Classification in MXNet Gluon This example provides an end-to-end pipeline for a common datahack competition - Urban Sounds Classification Example. diff --git a/example/gluon/dc_gan/README.md b/example/gluon/dc_gan/README.md index 5aacd78a3ed5..1d91d91a3452 100644 --- a/example/gluon/dc_gan/README.md +++ b/example/gluon/dc_gan/README.md @@ -1,3 +1,22 @@ + + # DCGAN in MXNet [Deep Convolutional Generative Adversarial Networks(DCGAN)](https://arxiv.org/abs/1511.06434) implementation with Apache MXNet GLUON. diff --git a/example/gluon/embedding_learning/README.md b/example/gluon/embedding_learning/README.md index e7821619a381..de11ea0db072 100644 --- a/example/gluon/embedding_learning/README.md +++ b/example/gluon/embedding_learning/README.md @@ -1,3 +1,22 @@ + + # Image Embedding Learning This example implements embedding learning based on a Margin-based Loss with distance weighted sampling [(Wu et al, 2017)](http://www.philkr.net/papers/2017-10-01-iccv/2017-10-01-iccv.pdf). The model obtains a validation Recall@1 of ~64% on the [Caltech-UCSD Birds-200-2011](http://www.vision.caltech.edu/visipedia/CUB-200-2011.html) dataset. diff --git a/example/gluon/sn_gan/README.md b/example/gluon/sn_gan/README.md index 5b2a750e4efb..60327bc959a6 100644 --- a/example/gluon/sn_gan/README.md +++ b/example/gluon/sn_gan/README.md @@ -1,3 +1,22 @@ + + # Spectral Normalization GAN This example implements [Spectral Normalization for Generative Adversarial Networks](https://arxiv.org/abs/1802.05957) based on [CIFAR10](https://www.cs.toronto.edu/~kriz/cifar.html) dataset. diff --git a/example/gluon/style_transfer/README.md b/example/gluon/style_transfer/README.md index ef273a5975ab..1162e7241053 100644 --- a/example/gluon/style_transfer/README.md +++ b/example/gluon/style_transfer/README.md @@ -1,3 +1,22 @@ + + # MXNet-Gluon-Style-Transfer This repo provides MXNet Implementation of **[Neural Style Transfer](#neural-style)** and **[MSG-Net](#real-time-style-transfer)**. diff --git a/example/gluon/tree_lstm/README.md b/example/gluon/tree_lstm/README.md index e14ab4c70afc..93d6289f42af 100644 --- a/example/gluon/tree_lstm/README.md +++ b/example/gluon/tree_lstm/README.md @@ -1,3 +1,22 @@ + + # Tree-Structured Long Short-Term Memory Networks This is a [MXNet Gluon](https://mxnet.io/) implementation of Tree-LSTM as described in the paper [Improved Semantic Representations From Tree-Structured Long Short-Term Memory Networks](http://arxiv.org/abs/1503.00075) by Kai Sheng Tai, Richard Socher, and Christopher Manning. diff --git a/example/gluon/word_language_model/README.md b/example/gluon/word_language_model/README.md index 4a77950d01bc..7c9b74ad3379 100644 --- a/example/gluon/word_language_model/README.md +++ b/example/gluon/word_language_model/README.md @@ -1,3 +1,22 @@ + + # Word-level language modeling RNN This example trains a multi-layer RNN (Elman, GRU, or LSTM) on WikiText-2 language modeling benchmark. diff --git a/example/kaggle-ndsb1/README.md b/example/kaggle-ndsb1/README.md index 82a99f7b6947..54d33b8b1df2 100644 --- a/example/kaggle-ndsb1/README.md +++ b/example/kaggle-ndsb1/README.md @@ -1,3 +1,22 @@ + + Tutorial for Kaggle NDSB-1 ----- diff --git a/example/kaggle-ndsb2/README.md b/example/kaggle-ndsb2/README.md index 302e54033c8a..d1e29d4f86bb 100644 --- a/example/kaggle-ndsb2/README.md +++ b/example/kaggle-ndsb2/README.md @@ -1,3 +1,22 @@ + + # End-to-End Deep Learning Tutorial for Kaggle NDSB-II In this example, we will demo how to use MXNet to build an end-to-end deep learning system to help Diagnose Heart Disease. The demo network is able to achieve 0.039222 CRPS on validation set, which is good enough to get Top-10 (on Dec 22nd, 2015). diff --git a/example/model-parallel/matrix_factorization/README.md b/example/model-parallel/matrix_factorization/README.md index 00507d924f81..ee5384565d45 100644 --- a/example/model-parallel/matrix_factorization/README.md +++ b/example/model-parallel/matrix_factorization/README.md @@ -1,3 +1,22 @@ + + Model Parallel Matrix Factorization =================================== diff --git a/example/module/README.md b/example/module/README.md index 99dd756ead63..8dd3d6d87c7c 100644 --- a/example/module/README.md +++ b/example/module/README.md @@ -1,3 +1,22 @@ + + # Module Usage Example This folder contains usage examples for MXNet module. diff --git a/example/multi-task/README.md b/example/multi-task/README.md index b7756fe378a7..4ca744189221 100644 --- a/example/multi-task/README.md +++ b/example/multi-task/README.md @@ -1,3 +1,22 @@ + + # Mulit-task learning example This is a simple example to show how to use mxnet for multi-task learning. It uses MNIST as an example, trying to predict jointly the digit and whether this digit is odd or even. diff --git a/example/multivariate_time_series/README.md b/example/multivariate_time_series/README.md index 87baca36d35f..6c9e1a451f10 100644 --- a/example/multivariate_time_series/README.md +++ b/example/multivariate_time_series/README.md @@ -1,3 +1,22 @@ + + # LSTNet - This repo contains an MXNet implementation of [this](https://arxiv.org/pdf/1703.07015.pdf) state of the art time series forecasting model. diff --git a/example/named_entity_recognition/README.md b/example/named_entity_recognition/README.md index c914a6985dfe..897ea4221b36 100644 --- a/example/named_entity_recognition/README.md +++ b/example/named_entity_recognition/README.md @@ -1,3 +1,22 @@ + + ## Goal - This repo contains an MXNet implementation of this state of the art [entity recognition model](https://www.aclweb.org/anthology/Q16-1026). diff --git a/example/nce-loss/README.md b/example/nce-loss/README.md index 56e43525a7ca..4a847e420732 100644 --- a/example/nce-loss/README.md +++ b/example/nce-loss/README.md @@ -1,3 +1,22 @@ + + # Examples of NCE Loss [Noise-contrastive estimation](http://proceedings.mlr.press/v9/gutmann10a/gutmann10a.pdf) loss (nce-loss) is used to speedup multi-class classification when class num is huge. diff --git a/example/neural-style/README.md b/example/neural-style/README.md index 5c4b58924827..0f9545cf65b6 100644 --- a/example/neural-style/README.md +++ b/example/neural-style/README.md @@ -1,3 +1,22 @@ + + # Neural art This is an implementation of the paper diff --git a/example/neural-style/end_to_end/README.md b/example/neural-style/end_to_end/README.md index 4a228c199bb7..0c38f427c300 100644 --- a/example/neural-style/end_to_end/README.md +++ b/example/neural-style/end_to_end/README.md @@ -1,3 +1,22 @@ + + # End to End Neural Art Please refer to this [blog](http://dmlc.ml/mxnet/2016/06/20/end-to-end-neural-style.html) for details of how it is implemented. diff --git a/example/numpy-ops/README.md b/example/numpy-ops/README.md index aa4911f67414..a1ed1a5573e4 100644 --- a/example/numpy-ops/README.md +++ b/example/numpy-ops/README.md @@ -1,3 +1,22 @@ + + # Training with Custom Operators in Python These examples demonstrate custom operator implementations in python. diff --git a/example/profiler/README.md b/example/profiler/README.md index 1b9279ccf227..14668fdbe0e5 100644 --- a/example/profiler/README.md +++ b/example/profiler/README.md @@ -1,3 +1,22 @@ + + # MXNet Profiler Examples This folder contains examples of using MXNet profiler to generate profiling results in json files. diff --git a/example/rcnn/README.md b/example/rcnn/README.md index 5e6127ccb08d..737ef4418914 100644 --- a/example/rcnn/README.md +++ b/example/rcnn/README.md @@ -1,3 +1,22 @@ + + # Faster R-CNN in MXNet Please redirect any issue or question of using this symbolic example of Faster R-CNN to https://github.com/ijkguo/mx-rcnn. diff --git a/example/recommenders/README.md b/example/recommenders/README.md index 628182c849b8..2534074268e8 100644 --- a/example/recommenders/README.md +++ b/example/recommenders/README.md @@ -1,3 +1,22 @@ + + # Recommender Systems diff --git a/example/reinforcement-learning/a3c/README.md b/example/reinforcement-learning/a3c/README.md index 5eaba66a5b86..1a8b4f3ba7a1 100644 --- a/example/reinforcement-learning/a3c/README.md +++ b/example/reinforcement-learning/a3c/README.md @@ -1,3 +1,22 @@ + + # A3C Implementation This is an attempt to implement the A3C algorithm in paper Asynchronous Methods for Deep Reinforcement Learning. diff --git a/example/reinforcement-learning/ddpg/README.md b/example/reinforcement-learning/ddpg/README.md index 2e299dd5daa3..c3040434823a 100644 --- a/example/reinforcement-learning/ddpg/README.md +++ b/example/reinforcement-learning/ddpg/README.md @@ -1,3 +1,22 @@ + + # mx-DDPG MXNet Implementation of DDPG diff --git a/example/reinforcement-learning/parallel_actor_critic/README.md b/example/reinforcement-learning/parallel_actor_critic/README.md index d3288492a611..767c67a7d72b 100644 --- a/example/reinforcement-learning/parallel_actor_critic/README.md +++ b/example/reinforcement-learning/parallel_actor_critic/README.md @@ -1,3 +1,22 @@ + + # 'Parallel Advantage-Actor Critic' Implementation This repo contains a MXNet implementation of a variant of the A3C algorithm from [Asynchronous Methods for Deep Reinforcement Learning](https://arxiv.org/pdf/1602.01783v2.pdf). diff --git a/example/restricted-boltzmann-machine/README.md b/example/restricted-boltzmann-machine/README.md index a8769a51e05a..30ee8cfb9c3d 100644 --- a/example/restricted-boltzmann-machine/README.md +++ b/example/restricted-boltzmann-machine/README.md @@ -1,3 +1,22 @@ + + # Restricted Boltzmann machine (RBM) An example of the binary RBM [1] learning the MNIST data. The RBM is implemented as a custom operator, and a gluon block is also provided. `binary_rbm.py` contains the implementation of the RBM. `binary_rbm_module.py` and `binary_rbm_gluon.py` train the MNIST data using the module interface and the gluon interface respectively. The MNIST data is downloaded automatically. diff --git a/example/rnn/README.md b/example/rnn/README.md index 1d1df6ed7687..a819dfa1097b 100644 --- a/example/rnn/README.md +++ b/example/rnn/README.md @@ -1,3 +1,22 @@ + + Recurrent Neural Network Examples =========== diff --git a/example/rnn/bucketing/README.md b/example/rnn/bucketing/README.md index 7b7883d79ad1..e06d9f64da27 100644 --- a/example/rnn/bucketing/README.md +++ b/example/rnn/bucketing/README.md @@ -1,3 +1,22 @@ + + RNN Example =========== This folder contains RNN examples using high level mxnet.rnn interface. diff --git a/example/rnn/old/README.md b/example/rnn/old/README.md index 5d73523dd964..806f50b834de 100644 --- a/example/rnn/old/README.md +++ b/example/rnn/old/README.md @@ -1,3 +1,22 @@ + + RNN Example =========== This folder contains RNN examples using low level symbol interface. diff --git a/example/rnn/word_lm/README.md b/example/rnn/word_lm/README.md index ab0a8d704b9c..dbb7832e15dc 100644 --- a/example/rnn/word_lm/README.md +++ b/example/rnn/word_lm/README.md @@ -1,3 +1,22 @@ + + Word Level Language Modeling =========== This example trains a multi-layer LSTM on Sherlock Holmes language modeling benchmark. diff --git a/example/sparse/factorization_machine/README.md b/example/sparse/factorization_machine/README.md index 32b956ed0201..42b93741fedb 100644 --- a/example/sparse/factorization_machine/README.md +++ b/example/sparse/factorization_machine/README.md @@ -1,3 +1,22 @@ + + Factorization Machine =========== This example trains a factorization machine model using the criteo dataset. diff --git a/example/sparse/linear_classification/README.md b/example/sparse/linear_classification/README.md index 926d9234269d..38fde454f1b2 100644 --- a/example/sparse/linear_classification/README.md +++ b/example/sparse/linear_classification/README.md @@ -1,3 +1,22 @@ + + Linear Classification Using Sparse Matrix Multiplication =========== This examples trains a linear model using the sparse feature in MXNet. This is for demonstration purpose only. diff --git a/example/sparse/matrix_factorization/README.md b/example/sparse/matrix_factorization/README.md index ddbf662c858f..d11219771012 100644 --- a/example/sparse/matrix_factorization/README.md +++ b/example/sparse/matrix_factorization/README.md @@ -1,3 +1,22 @@ + + Matrix Factorization w/ Sparse Embedding =========== The example demonstrates the basic usage of the sparse.Embedding operator in MXNet, adapted based on @leopd's recommender examples. diff --git a/example/sparse/wide_deep/README.md b/example/sparse/wide_deep/README.md index 3df5e420ee36..80924274e032 100644 --- a/example/sparse/wide_deep/README.md +++ b/example/sparse/wide_deep/README.md @@ -1,3 +1,22 @@ + + ## Wide and Deep Learning The example demonstrates how to train [wide and deep model](https://arxiv.org/abs/1606.07792). The [Census Income Data Set](https://archive.ics.uci.edu/ml/datasets/Census+Income) that this example uses for training is hosted by the [UC Irvine Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/). Tricks of feature engineering are adapted from tensorflow's [wide and deep tutorial](https://github.com/tensorflow/models/tree/master/official/wide_deep). diff --git a/example/speech_recognition/README.md b/example/speech_recognition/README.md index 6f01911e1300..3dad016a48d8 100644 --- a/example/speech_recognition/README.md +++ b/example/speech_recognition/README.md @@ -1,3 +1,22 @@ + + **deepSpeech.mxnet: Rich Speech Example** ========================================= diff --git a/example/ssd/README.md b/example/ssd/README.md index 713a9ea33c1b..753c967aeaf1 100644 --- a/example/ssd/README.md +++ b/example/ssd/README.md @@ -1,3 +1,22 @@ + + # SSD: Single Shot MultiBox Object Detector SSD is an unified framework for object detection with a single network. diff --git a/example/ssd/dataset/pycocotools/README.md b/example/ssd/dataset/pycocotools/README.md old mode 100755 new mode 100644 index d358f53105da..0e10d4080aac --- a/example/ssd/dataset/pycocotools/README.md +++ b/example/ssd/dataset/pycocotools/README.md @@ -1,2 +1,21 @@ + + This is a modified version of https://github.com/pdollar/coco python API. No `make` is required, but this will not support mask functions. diff --git a/example/ssd/model/README.md b/example/ssd/model/README.md index e5bac52f5a83..0ee70c3d1fff 100644 --- a/example/ssd/model/README.md +++ b/example/ssd/model/README.md @@ -1 +1,20 @@ + + #### This is the default directory to store all the models, including `*.params` and `*.json` diff --git a/example/ssd/symbol/README.md b/example/ssd/symbol/README.md index 8fee31985a0d..67f64f584f8b 100644 --- a/example/ssd/symbol/README.md +++ b/example/ssd/symbol/README.md @@ -1,3 +1,22 @@ + + ## How to compose SSD network on top of mainstream classification networks 1. Have the base network ready in this directory as `name.py`, such as `inceptionv3.py`. diff --git a/example/ssd/tools/caffe_converter/README.md b/example/ssd/tools/caffe_converter/README.md index 2e74fc56e022..9eb59812e207 100644 --- a/example/ssd/tools/caffe_converter/README.md +++ b/example/ssd/tools/caffe_converter/README.md @@ -1,3 +1,22 @@ + + # Convert Caffe Model to Mxnet Format This folder contains the source codes for this tool. diff --git a/example/stochastic-depth/README.md b/example/stochastic-depth/README.md index 08c466eb8b0e..a5dfcd9c8e42 100644 --- a/example/stochastic-depth/README.md +++ b/example/stochastic-depth/README.md @@ -1,3 +1,22 @@ + + Stochastic Depth ================ diff --git a/example/svm_mnist/README.md b/example/svm_mnist/README.md index 408f5108b44a..ffcf5c38bbd7 100644 --- a/example/svm_mnist/README.md +++ b/example/svm_mnist/README.md @@ -1,3 +1,22 @@ + + # Use case with Support Vector Machine To ensure that not only the implementation is learning, but is able to outsmart the softmax, as [this article](https://arxiv.org/pdf/1306.0239.pdf) suggests, I ran svm_mnist.py script. It was based on the MNIST experiment description on the article and [this tutorial](https://github.com/dmlc/mxnet-gtc-tutorial/blob/master/tutorial.ipynb). diff --git a/example/svrg_module/README.md b/example/svrg_module/README.md index 250995a57152..624b50d60f12 100644 --- a/example/svrg_module/README.md +++ b/example/svrg_module/README.md @@ -1,3 +1,22 @@ + + ## SVRGModule Example SVRGModule is an extension to the Module API that implements SVRG optimization, which stands for Stochastic diff --git a/example/vae-gan/README.md b/example/vae-gan/README.md index 469668b9b374..5d7ae272b32b 100644 --- a/example/vae-gan/README.md +++ b/example/vae-gan/README.md @@ -1,3 +1,22 @@ + + # VAE-GAN in MXNet * Implementation of [Autoencoding beyond pixels using a learned similarity metric](https://arxiv.org/abs/1512.09300), diff --git a/julia/LICENSE.md b/julia/LICENSE.md index 5ecf95ac60bc..4161166b7280 100644 --- a/julia/LICENSE.md +++ b/julia/LICENSE.md @@ -1,3 +1,22 @@ + + The MXNet.jl package is licensed under version 2.0 of the Apache License: > Copyright (c) 2015-2018: diff --git a/julia/NEWS.md b/julia/NEWS.md index 3da119496fac..1c8cda7176c6 100644 --- a/julia/NEWS.md +++ b/julia/NEWS.md @@ -1,3 +1,22 @@ + + # v1.5.0 (#TBD) * Following material from `mx` module got exported (#TBD): diff --git a/julia/README-DEV.md b/julia/README-DEV.md index a1d6fa9012fc..9edb8309c0c8 100644 --- a/julia/README-DEV.md +++ b/julia/README-DEV.md @@ -1,3 +1,22 @@ + + # Workflow for making a release 1. Update `NEWS.md` to list important changes diff --git a/julia/README.md b/julia/README.md index 91a3981464be..97a52ae9a4e6 100644 --- a/julia/README.md +++ b/julia/README.md @@ -1,3 +1,22 @@ + + # MXNet [![MXNet](http://pkg.julialang.org/badges/MXNet_0.6.svg)](http://pkg.julialang.org/?pkg=MXNet) diff --git a/julia/docs/src/api.md b/julia/docs/src/api.md index 4984129863d0..92c04581c7e8 100644 --- a/julia/docs/src/api.md +++ b/julia/docs/src/api.md @@ -1,3 +1,22 @@ + + # API Documentation ```@contents diff --git a/julia/docs/src/api/callback.md b/julia/docs/src/api/callback.md index f67811cc41fe..327014d7dbfc 100644 --- a/julia/docs/src/api/callback.md +++ b/julia/docs/src/api/callback.md @@ -1,3 +1,22 @@ + + # Callback in training ```@autodocs diff --git a/julia/docs/src/api/context.md b/julia/docs/src/api/context.md index 93ccf83e51ba..e82504663b06 100644 --- a/julia/docs/src/api/context.md +++ b/julia/docs/src/api/context.md @@ -1,3 +1,22 @@ + + # Context ```@autodocs diff --git a/julia/docs/src/api/executor.md b/julia/docs/src/api/executor.md index b560c7a0864d..81ce7fa17ef7 100644 --- a/julia/docs/src/api/executor.md +++ b/julia/docs/src/api/executor.md @@ -1,3 +1,22 @@ + + # Executor ```@autodocs diff --git a/julia/docs/src/api/initializer.md b/julia/docs/src/api/initializer.md index d0aad2def4cd..8f55b6e1da51 100644 --- a/julia/docs/src/api/initializer.md +++ b/julia/docs/src/api/initializer.md @@ -1,3 +1,22 @@ + + # Initializer ```@autodocs diff --git a/julia/docs/src/api/io.md b/julia/docs/src/api/io.md index 7312259dbf3c..bd53d2ca667e 100644 --- a/julia/docs/src/api/io.md +++ b/julia/docs/src/api/io.md @@ -1,3 +1,22 @@ + + # Data Providers Data providers are wrappers that load external data, be it images, text, or general tensors, diff --git a/julia/docs/src/api/kvstore.md b/julia/docs/src/api/kvstore.md index 34a5027f85fb..a8407b54bd34 100644 --- a/julia/docs/src/api/kvstore.md +++ b/julia/docs/src/api/kvstore.md @@ -1,3 +1,22 @@ + + # Key-Value Store ```@autodocs diff --git a/julia/docs/src/api/metric.md b/julia/docs/src/api/metric.md index 63cca0cc41ba..a31bc92e49e6 100644 --- a/julia/docs/src/api/metric.md +++ b/julia/docs/src/api/metric.md @@ -1,3 +1,22 @@ + + # Evaluation Metrics Evaluation metrics provide a way to evaluate the performance of a learned model. diff --git a/julia/docs/src/api/model.md b/julia/docs/src/api/model.md index f793c7c406c7..8240aeb022bb 100644 --- a/julia/docs/src/api/model.md +++ b/julia/docs/src/api/model.md @@ -1,3 +1,22 @@ + + # Model The model API provides convenient high-level interface to do training and predicting on diff --git a/julia/docs/src/api/ndarray.md b/julia/docs/src/api/ndarray.md index 5877d8257758..41161d547881 100644 --- a/julia/docs/src/api/ndarray.md +++ b/julia/docs/src/api/ndarray.md @@ -1,3 +1,22 @@ + + # NDArray API ## Arithmetic Operations diff --git a/julia/docs/src/api/nn-factory.md b/julia/docs/src/api/nn-factory.md index 833d9a3efd53..d8106eff158f 100644 --- a/julia/docs/src/api/nn-factory.md +++ b/julia/docs/src/api/nn-factory.md @@ -1,3 +1,22 @@ + + # Neural Network Factory Neural network factory provide convenient helper functions to define diff --git a/julia/docs/src/api/symbolic-node.md b/julia/docs/src/api/symbolic-node.md index ef731d9f7d00..c23b67602627 100644 --- a/julia/docs/src/api/symbolic-node.md +++ b/julia/docs/src/api/symbolic-node.md @@ -1,3 +1,22 @@ + + # Symbolic API ```@autodocs diff --git a/julia/docs/src/api/visualize.md b/julia/docs/src/api/visualize.md index 429a927012e4..843922420a30 100644 --- a/julia/docs/src/api/visualize.md +++ b/julia/docs/src/api/visualize.md @@ -1,3 +1,22 @@ + + # Network Visualization ```@autodocs diff --git a/julia/docs/src/index.md b/julia/docs/src/index.md index b6a51fc162ad..8274c712e549 100644 --- a/julia/docs/src/index.md +++ b/julia/docs/src/index.md @@ -1,3 +1,22 @@ + + # MXNet Documentation [MXNet.jl](https://github.com/dmlc/MXNet.jl) is the diff --git a/julia/docs/src/tutorial/char-lstm.md b/julia/docs/src/tutorial/char-lstm.md index 369bcddd53e9..53a371e028fe 100644 --- a/julia/docs/src/tutorial/char-lstm.md +++ b/julia/docs/src/tutorial/char-lstm.md @@ -1,3 +1,22 @@ + + Generating Random Sentence with LSTM RNN ======================================== diff --git a/julia/docs/src/tutorial/mnist.md b/julia/docs/src/tutorial/mnist.md index 916e46deb853..96423266db1b 100644 --- a/julia/docs/src/tutorial/mnist.md +++ b/julia/docs/src/tutorial/mnist.md @@ -1,3 +1,22 @@ + + Digit Recognition on MNIST ========================== diff --git a/julia/docs/src/user-guide/faq.md b/julia/docs/src/user-guide/faq.md index 8fd8a6b34551..d288ad0b3b8e 100644 --- a/julia/docs/src/user-guide/faq.md +++ b/julia/docs/src/user-guide/faq.md @@ -1,3 +1,22 @@ + + FAQ === diff --git a/julia/docs/src/user-guide/install.md b/julia/docs/src/user-guide/install.md index f1d5eeefacfe..52628de7a255 100644 --- a/julia/docs/src/user-guide/install.md +++ b/julia/docs/src/user-guide/install.md @@ -1,3 +1,22 @@ + + Installation Guide ================== diff --git a/julia/docs/src/user-guide/overview.md b/julia/docs/src/user-guide/overview.md index 5815bc6d772c..f0189f7bcd2e 100644 --- a/julia/docs/src/user-guide/overview.md +++ b/julia/docs/src/user-guide/overview.md @@ -1,3 +1,22 @@ + + # Overview ## MXNet.jl Namespace diff --git a/julia/examples/char-lstm/README.md b/julia/examples/char-lstm/README.md index ff16ee0a3ae9..ac77e15b131f 100644 --- a/julia/examples/char-lstm/README.md +++ b/julia/examples/char-lstm/README.md @@ -1,3 +1,22 @@ + + # LSTM char-rnn Because we explicitly unroll the LSTM/RNN over time for a fixed sequence length, diff --git a/julia/plugins/README.md b/julia/plugins/README.md index 38882889f494..c5ca926ca0ac 100644 --- a/julia/plugins/README.md +++ b/julia/plugins/README.md @@ -1,3 +1,22 @@ + + # Plugins of MXNet.jl This directory contains *plugins* of MXNet.jl. A plugin is typically a component that could be part of MXNet.jl, but excluded from the `mx` namespace. The plugins are included here primarily for two reasons: diff --git a/matlab/README.md b/matlab/README.md index 939b7011a4f2..13a83922d915 100644 --- a/matlab/README.md +++ b/matlab/README.md @@ -1,3 +1,22 @@ + + # MATLAB binding for MXNet ### How to use diff --git a/perl-package/AI-MXNet/examples/gluon/style_transfer/README.md b/perl-package/AI-MXNet/examples/gluon/style_transfer/README.md index 658a77530a92..3da1f97b517c 100644 --- a/perl-package/AI-MXNet/examples/gluon/style_transfer/README.md +++ b/perl-package/AI-MXNet/examples/gluon/style_transfer/README.md @@ -1,3 +1,22 @@ + + This directory provides AI::MXNet Implementation of MSG-Net real time style transfer, https://arxiv.org/abs/1703.06953 ### Stylize Images Using Pre-trained MSG-Net diff --git a/perl-package/AI-MXNet/examples/sparse/matrix_factorization/README.md b/perl-package/AI-MXNet/examples/sparse/matrix_factorization/README.md index debad272206f..e4c89db760f4 100644 --- a/perl-package/AI-MXNet/examples/sparse/matrix_factorization/README.md +++ b/perl-package/AI-MXNet/examples/sparse/matrix_factorization/README.md @@ -1,3 +1,22 @@ + + Matrix Factorization w/ Sparse Embedding =========== The example demonstrates the basic usage of the SparseEmbedding operator in MXNet, adapted based on @leopd's recommender examples. diff --git a/perl-package/AI-MXNet/examples/sparse/wide_deep/README.md b/perl-package/AI-MXNet/examples/sparse/wide_deep/README.md index 9a481d69edf2..fc3192623de4 100644 --- a/perl-package/AI-MXNet/examples/sparse/wide_deep/README.md +++ b/perl-package/AI-MXNet/examples/sparse/wide_deep/README.md @@ -1,3 +1,22 @@ + + ## Wide and Deep Learning The example demonstrates how to train [wide and deep model](https://arxiv.org/abs/1606.07792). The [Census Income Data Set](https://archive.ics.uci.edu/ml/datasets/Census+Income) that this example uses for training is hosted by the [UC Irvine Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/). Tricks of feature engineering are adapted from tensorflow's [wide and deep tutorial](https://github.com/tensorflow/models/tree/master/official/wide_deep). diff --git a/perl-package/README.md b/perl-package/README.md index 93e34d1af37a..20ce7e635e6f 100644 --- a/perl-package/README.md +++ b/perl-package/README.md @@ -1,3 +1,22 @@ + + [Perl API](https://mxnet.incubator.apache.org/api/perl/index.html) [![GitHub license](http://dmlc.github.io/img/apache2.svg)](../LICENSE) diff --git a/plugin/caffe/README.md b/plugin/caffe/README.md index 466305cc9b88..6541d4dacc04 100644 --- a/plugin/caffe/README.md +++ b/plugin/caffe/README.md @@ -1,3 +1,22 @@ + + # How to use Caffe operator in MXNet [Caffe](http://caffe.berkeleyvision.org/) has been a well-known and widely-used deep learning framework. Now MXNet has supported calling most caffe operators(layers) and loss functions directly in its symbolic graph! Using one's own customized caffe layer is also effortless. diff --git a/python/README.md b/python/README.md index 1ab7aa4464a3..4e180360f674 100644 --- a/python/README.md +++ b/python/README.md @@ -1,3 +1,22 @@ + + MXNet Python Package ==================== This directory and nested files contain MXNet Python package and language binding. diff --git a/python/minpy/README.md b/python/minpy/README.md deleted file mode 100644 index 4f028e3b21ad..000000000000 --- a/python/minpy/README.md +++ /dev/null @@ -1,4 +0,0 @@ -MXNet Python Package -==================== - -This is the WIP directory for MinPy project. diff --git a/scala-package/README.md b/scala-package/README.md index be0fc41a5fe4..7dd5f5ea0680 100644 --- a/scala-package/README.md +++ b/scala-package/README.md @@ -1,3 +1,22 @@ + + MXNet Package for Scala/Java ===== diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/objectdetector/README.md b/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/objectdetector/README.md index 55741024d08b..b6c92c1204fa 100644 --- a/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/objectdetector/README.md +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/objectdetector/README.md @@ -1,3 +1,22 @@ + + # Single Shot Multi Object Detection using Java Inference API In this example, you will learn how to use Java Inference API to run Inference on pre-trained Single Shot Multi Object Detection (SSD) MXNet model. diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/predictor/README.md b/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/predictor/README.md index 1f2c9e0e813c..cfad6a4e9a6e 100644 --- a/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/predictor/README.md +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/predictor/README.md @@ -1,3 +1,22 @@ + + # Image Classification using Java Predictor In this example, you will learn how to use Java Inference API to diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md index 753cb3125410..efeab0a188cf 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md @@ -1,3 +1,22 @@ + + # Benchmarking Scala Inference APIs This folder contains a base class [ScalaInferenceBenchmark](https://github.com/apache/incubator-mxnet/tree/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/) and provides a mechanism for benchmarking [MXNet Inference APIs]((https://github.com/apache/incubator-mxnet/tree/master/scala-package/infer)) in Scala. diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md index 5e3602e8ab15..ae2b68002e3c 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md @@ -1,3 +1,22 @@ + + # CNN Text Classification Example for Scala This is the example using Scala type-safe api doing CNN text classification. This example is only for Illustration and not modeled to achieve the best accuracy. diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md index 886fa2cc9d46..bf2429399e94 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md @@ -1,3 +1,22 @@ + + # Custom Operator Example for Scala This is the example using Custom Operator for type-safe api of Scala. In the example, a `Softmax` operator is implemented to run the MNIST example. diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/gan/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/gan/README.md index 40db092727c4..fd477c56f54e 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/gan/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/gan/README.md @@ -1,3 +1,22 @@ + + # GAN MNIST Example for Scala This is the GAN MNIST Training Example implemented for Scala type-safe api diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/imclassification/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/imclassification/README.md index cec750acdc92..55e065e1f493 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/imclassification/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/imclassification/README.md @@ -1,3 +1,22 @@ + + # Image Classification Models This examples contains a number of image classification models that can be run on various datasets. diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md index 541e0ce8dd31..5e8a51789300 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md @@ -1,3 +1,22 @@ + + # Image Classification This folder contains an example for image classification with the [MXNet Scala Infer API](https://github.com/apache/incubator-mxnet/tree/master/scala-package/infer). diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md index 77aec7bb5dee..e3190b2fbcbf 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md @@ -1,3 +1,22 @@ + + # Single Shot Multi Object Detection using Scala Inference API In this example, you will learn how to use Scala Inference API to run Inference on pre-trained Single Shot Multi Object Detection (SSD) MXNet model. diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/neuralstyle/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/neuralstyle/README.md index fe849343c9d7..2d39a9c67733 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/neuralstyle/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/neuralstyle/README.md @@ -1,3 +1,22 @@ + + # Neural Style Example for Scala ## Introduction diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/README.md index 5289fc7b1b4e..dea2c12667e9 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/README.md @@ -1,3 +1,22 @@ + + # RNN Example for MXNet Scala This folder contains the following examples writing in new Scala type-safe API: - [x] LSTM Bucketing diff --git a/scala-package/memory-management.md b/scala-package/memory-management.md index 33c36b6e6ab0..e9a0c71ded02 100644 --- a/scala-package/memory-management.md +++ b/scala-package/memory-management.md @@ -1,3 +1,22 @@ + + # JVM Memory Management The Scala and Java bindings of Apache MXNet use native memory (memory from the C++ heap in either RAM or GPU memory) for most of the MXNet objects such as NDArray, Symbol, Executor, KVStore, Data Iterators, etc. The associated Scala classes act only as wrappers. The operations done on these wrapper objects are then directed to the high performance MXNet C++ backend via the Java Native Interface (JNI). Therefore, the bytes are stored in the C++ native heap which allows for fast access. diff --git a/scala-package/mxnet-demo/java-demo/README.md b/scala-package/mxnet-demo/java-demo/README.md index 5dfbf14e8df2..3d5041c228f6 100644 --- a/scala-package/mxnet-demo/java-demo/README.md +++ b/scala-package/mxnet-demo/java-demo/README.md @@ -1,3 +1,22 @@ + + # MXNet Java Sample Project This is an project created to use Maven-published Scala/Java package with two Java examples. ## Setup diff --git a/scala-package/mxnet-demo/scala-demo/README.md b/scala-package/mxnet-demo/scala-demo/README.md index b994a196b1f6..8a5abf56894b 100644 --- a/scala-package/mxnet-demo/scala-demo/README.md +++ b/scala-package/mxnet-demo/scala-demo/README.md @@ -1,3 +1,22 @@ + + # MXNet Scala Sample Project This is an project created to use Maven-published Scala package with two Scala examples. ## Setup diff --git a/scala-package/native/README.md b/scala-package/native/README.md index c87b064fff02..17e913f1a4eb 100644 --- a/scala-package/native/README.md +++ b/scala-package/native/README.md @@ -1,3 +1,22 @@ + + # MXNet Scala JNI MXNet Scala JNI is a thin wrapper layer of underlying libmxnet.so. diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md index e9980f353759..f14cdc09c180 100644 --- a/scala-package/packageTest/README.md +++ b/scala-package/packageTest/README.md @@ -1,3 +1,22 @@ + + # MXNet Scala Package Test This is an project created to run the test suite on a fully packaged mxnet jar. The test suite is found locally but mxnet is from the target jarfile. diff --git a/scala-package/spark/README.md b/scala-package/spark/README.md index 503c279038a5..79f637a3dfbf 100644 --- a/scala-package/spark/README.md +++ b/scala-package/spark/README.md @@ -1,3 +1,22 @@ + + Deep Learning on Spark ===== diff --git a/tests/README.md b/tests/README.md index e528edf2a9da..be09cadff3ca 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,3 +1,22 @@ + + # Testing MXNET ## Running CPP Tests diff --git a/tests/nightly/README.md b/tests/nightly/README.md old mode 100755 new mode 100644 index fa1771a7eeb0..5cf41eb2c3ef --- a/tests/nightly/README.md +++ b/tests/nightly/README.md @@ -1,3 +1,22 @@ + + # Nightly Tests for MXNet These are some longer running tests that are scheduled to run every night. diff --git a/tests/nightly/apache_rat_license_check/README.md b/tests/nightly/apache_rat_license_check/README.md old mode 100755 new mode 100644 index e8578a857224..0d6d37dc7da8 --- a/tests/nightly/apache_rat_license_check/README.md +++ b/tests/nightly/apache_rat_license_check/README.md @@ -1,3 +1,22 @@ + + # Apache RAT License Check This is a nightly test that runs the Apache Tool RAT to check the License Headers on all source files diff --git a/tests/nightly/apache_rat_license_check/rat-excludes b/tests/nightly/apache_rat_license_check/rat-excludes index 6e7ae8c1bfdc..b3f64c246e6d 100755 --- a/tests/nightly/apache_rat_license_check/rat-excludes +++ b/tests/nightly/apache_rat_license_check/rat-excludes @@ -5,18 +5,15 @@ .*html .*json .*txt -.*md 3rdparty/* R-package/* trunk/* -docker/* .*\\.m .*\\.mk .*\\.R .*svg .*cfg .*config -docs/* __init__.py build/* .*\\.t @@ -38,7 +35,6 @@ erfinv-inl.h im2col.cuh im2col.h pool.h -README.rst dataset.cPickle image-classification/* rat-excludes @@ -51,3 +47,5 @@ Project.toml include/* .*.iml .*.json.ref +searchtools_custom.js +theme.conf diff --git a/tests/nightly/broken_link_checker_test/README.md b/tests/nightly/broken_link_checker_test/README.md old mode 100755 new mode 100644 index c39abd0d6175..aaad68601798 --- a/tests/nightly/broken_link_checker_test/README.md +++ b/tests/nightly/broken_link_checker_test/README.md @@ -1,3 +1,22 @@ + + # Broken link checker test This folder contains the scripts that are required to run the nightly job of checking the broken links. diff --git a/tests/nightly/model_backwards_compatibility_check/README.md b/tests/nightly/model_backwards_compatibility_check/README.md index 7a2116ac564e..af17396f0e0f 100644 --- a/tests/nightly/model_backwards_compatibility_check/README.md +++ b/tests/nightly/model_backwards_compatibility_check/README.md @@ -1,3 +1,22 @@ + + # Model Backwards Compatibility Tests This folder contains the scripts that are required to run the nightly job of verifying the compatibility and inference results of models (trained on earlier versions of MXNet) when loaded on the latest release candidate. The tests flag if: diff --git a/tests/nightly/straight_dope/README.md b/tests/nightly/straight_dope/README.md old mode 100755 new mode 100644 index 65a615b58d7e..869d80afcdfa --- a/tests/nightly/straight_dope/README.md +++ b/tests/nightly/straight_dope/README.md @@ -1,3 +1,22 @@ + + # Nightly Tests for MXNet: The Straight Dope These are some longer running tests that are scheduled to run every night. diff --git a/tests/python/README.md b/tests/python/README.md index 02dcb6ea6818..fd2282d67b9d 100644 --- a/tests/python/README.md +++ b/tests/python/README.md @@ -1,3 +1,22 @@ + + Python Test Case ================ This folder contains test cases for mxnet in python. diff --git a/tools/accnn/README.md b/tools/accnn/README.md index 02f10d111e2d..ca6d735bba39 100644 --- a/tools/accnn/README.md +++ b/tools/accnn/README.md @@ -1,3 +1,22 @@ + + # Accelerate Convolutional Neural Networks This tool aims to accelerate the test-time computation and decrease number of parameters of deep CNNs. diff --git a/tools/bandwidth/README.md b/tools/bandwidth/README.md index f087af7fd147..f82e3218a19c 100644 --- a/tools/bandwidth/README.md +++ b/tools/bandwidth/README.md @@ -1,3 +1,22 @@ + + # Measure communication bandwidth MXNet provides multiple ways to communicate data. The best choice depends on diff --git a/tools/caffe_converter/README.md b/tools/caffe_converter/README.md index d8ffc5cb83e5..b97b6e42ee5c 100644 --- a/tools/caffe_converter/README.md +++ b/tools/caffe_converter/README.md @@ -1,3 +1,22 @@ + + # Convert Caffe Model to Mxnet Format This folder contains the source codes for this tool. diff --git a/tools/caffe_translator/README.md b/tools/caffe_translator/README.md index ad111617b7ed..5d80caea288e 100644 --- a/tools/caffe_translator/README.md +++ b/tools/caffe_translator/README.md @@ -1,3 +1,22 @@ + + # Caffe Translator Caffe Translator is a migration tool that helps developers migrate their existing Caffe code to MXNet and continue further development using MXNet. Note that this is different from the Caffe to MXNet model converter which is available [here](https://github.com/apache/incubator-mxnet/tree/master/tools/caffe_converter). diff --git a/tools/caffe_translator/build_from_source.md b/tools/caffe_translator/build_from_source.md index 09af64e41460..f51d2fcaf3f0 100644 --- a/tools/caffe_translator/build_from_source.md +++ b/tools/caffe_translator/build_from_source.md @@ -1,3 +1,22 @@ + + ### Build Caffe Translator from source #### Prerequisites: diff --git a/tools/caffe_translator/faq.md b/tools/caffe_translator/faq.md index 99d19fef500b..186c0f623a14 100644 --- a/tools/caffe_translator/faq.md +++ b/tools/caffe_translator/faq.md @@ -1,3 +1,22 @@ + + ### Frequently asked questions [**Why is Caffe required to run the translated code?**](#why_caffe) diff --git a/tools/cfn/Readme.md b/tools/cfn/Readme.md index 677a1826fbb7..ecbdf836c9ef 100644 --- a/tools/cfn/Readme.md +++ b/tools/cfn/Readme.md @@ -1,2 +1,21 @@ + + **Distributed Deep Learning Made Easy has found more love and new home, please visit [awslabs/deeplearning-cfn](https://github.com/awslabs/deeplearning-cfn)** \ No newline at end of file diff --git a/tools/coreml/README.md b/tools/coreml/README.md index 45f19b608bdb..87f0a953dc71 100644 --- a/tools/coreml/README.md +++ b/tools/coreml/README.md @@ -1,3 +1,22 @@ + + # Convert MXNet models into Apple CoreML format. This tool helps convert MXNet models into [Apple CoreML](https://developer.apple.com/documentation/coreml) format which can then be run on Apple devices. diff --git a/tools/coreml/pip_package/README.rst b/tools/coreml/pip_package/README.rst index 875d89fcd208..c2e66f708e85 100644 --- a/tools/coreml/pip_package/README.rst +++ b/tools/coreml/pip_package/README.rst @@ -1,3 +1,20 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + MXNET -> CoreML Converter ========================= diff --git a/tools/dependencies/README.md b/tools/dependencies/README.md index c30e85d519c3..c8a06d868b44 100644 --- a/tools/dependencies/README.md +++ b/tools/dependencies/README.md @@ -1,3 +1,22 @@ + + # Overview This folder contains scripts for building the dependencies from source. The static libraries from diff --git a/tools/staticbuild/README.md b/tools/staticbuild/README.md index 2def768a1f1e..3297bbdfbd40 100644 --- a/tools/staticbuild/README.md +++ b/tools/staticbuild/README.md @@ -1,3 +1,22 @@ + + # MXNet Static Build This folder contains the core script used to build the static library. This README provides information on how to use the scripts in this folder. Please be aware, all of the scripts are designed to be run under the root folder. From 7ff6ad1586b21256a27aaebb34b6ac84e6db0e0b Mon Sep 17 00:00:00 2001 From: "seujung hwan, Jung" Date: Thu, 14 Feb 2019 03:23:25 +0900 Subject: [PATCH 12/75] Update lip reading example (#13647) * update lipnet * update utils * Update example/gluon/lipnet/README.md Co-Authored-By: seujung * Update example/gluon/lipnet/README.md Co-Authored-By: seujung * Update example/gluon/lipnet/utils/multi.py Co-Authored-By: seujung * Update example/gluon/lipnet/utils/preprocess_data.py Co-Authored-By: seujung * Update example/gluon/lipnet/utils/multi.py Co-Authored-By: seujung * Update example/gluon/lipnet/utils/download_data.py Co-Authored-By: seujung * fix error for using gpu mode * Add requirements * Remove unnecessary requirements * Update .gitignore * Remove inappropriate license file * Changed relative path * Fix description * Fix description * Fix description * Fix description * Change doc strings and add url reference * Fix align_path * Remove zip files * Fix bugs: source_path, n_process * Fix target_path * Fix exception handler and resume the preprocess * Pass the output when it fails to detect the mouth * Add exception during collecting images * Add the disk space and fix default align_path * Change optimizer * Update readme for pip * Update README * Add checkpoint folder * Apply to train using multiprocess * update network.py * delete batchnorm comment *fix dropout * fix loading ndarray as F * add space * Update readme * Add the info of GRID Data * Add the info of word alignments * Add total download size * Add time for preprocessing * Add test code for beamsearch * add space * delete line and fix code * Add shebang in BeamSearch * Fix trainer * Add space line * Fix appeding losses * Fix trainer * Delete debug line in data_loader * Move transpose of input into data_loader * Delete trailing-whitespace * Hybridize lip model * Hybridize model * Refactor the len of input sequence * Fix the shape of model * Apply to split train and validation * Split data into train and valid * Update Readme * Add infer.py * Remove ipynb * Apply to continual learning * Add images * Update readme * Fix typo and pylint * Fix loss digits of save_file and typo * Add info of data split and batch size --- example/gluon/lipnet/.gitignore | 3 + example/gluon/lipnet/BeamSearch.py | 170 +++++++++ example/gluon/lipnet/README.md | 194 ++++++++++ example/gluon/lipnet/asset/mouth_000.png | Bin 0 -> 6372 bytes example/gluon/lipnet/asset/mouth_001.png | Bin 0 -> 6826 bytes example/gluon/lipnet/asset/mouth_074.png | Bin 0 -> 6864 bytes .../gluon/lipnet/asset/network_structure.png | Bin 0 -> 183728 bytes example/gluon/lipnet/asset/s2_bbbf7p_000.png | Bin 0 -> 35141 bytes example/gluon/lipnet/asset/s2_bbbf7p_001.png | Bin 0 -> 36768 bytes example/gluon/lipnet/asset/s2_bbbf7p_074.png | Bin 0 -> 38248 bytes example/gluon/lipnet/checkpoint/__init__.py | 16 + example/gluon/lipnet/data_loader.py | 94 +++++ example/gluon/lipnet/infer.py | 52 +++ example/gluon/lipnet/main.py | 47 +++ example/gluon/lipnet/models/__init__.py | 0 example/gluon/lipnet/models/network.py | 73 ++++ example/gluon/lipnet/requirements.txt | 7 + example/gluon/lipnet/tests/test_beamsearch.py | 42 ++ example/gluon/lipnet/trainer.py | 232 +++++++++++ example/gluon/lipnet/utils/__init__.py | 16 + example/gluon/lipnet/utils/align.py | 83 ++++ example/gluon/lipnet/utils/common.py | 80 ++++ example/gluon/lipnet/utils/download_data.py | 112 ++++++ example/gluon/lipnet/utils/multi.py | 104 +++++ example/gluon/lipnet/utils/preprocess_data.py | 262 +++++++++++++ .../gluon/lipnet/utils/run_preprocess.ipynb | 194 ++++++++++ .../utils/run_preprocess_single_process.ipynb | 360 ++++++++++++++++++ 27 files changed, 2141 insertions(+) create mode 100644 example/gluon/lipnet/.gitignore create mode 100644 example/gluon/lipnet/BeamSearch.py create mode 100644 example/gluon/lipnet/README.md create mode 100644 example/gluon/lipnet/asset/mouth_000.png create mode 100644 example/gluon/lipnet/asset/mouth_001.png create mode 100644 example/gluon/lipnet/asset/mouth_074.png create mode 100644 example/gluon/lipnet/asset/network_structure.png create mode 100644 example/gluon/lipnet/asset/s2_bbbf7p_000.png create mode 100644 example/gluon/lipnet/asset/s2_bbbf7p_001.png create mode 100644 example/gluon/lipnet/asset/s2_bbbf7p_074.png create mode 100644 example/gluon/lipnet/checkpoint/__init__.py create mode 100644 example/gluon/lipnet/data_loader.py create mode 100644 example/gluon/lipnet/infer.py create mode 100644 example/gluon/lipnet/main.py create mode 100644 example/gluon/lipnet/models/__init__.py create mode 100644 example/gluon/lipnet/models/network.py create mode 100644 example/gluon/lipnet/requirements.txt create mode 100644 example/gluon/lipnet/tests/test_beamsearch.py create mode 100644 example/gluon/lipnet/trainer.py create mode 100644 example/gluon/lipnet/utils/__init__.py create mode 100644 example/gluon/lipnet/utils/align.py create mode 100644 example/gluon/lipnet/utils/common.py create mode 100644 example/gluon/lipnet/utils/download_data.py create mode 100644 example/gluon/lipnet/utils/multi.py create mode 100644 example/gluon/lipnet/utils/preprocess_data.py create mode 100644 example/gluon/lipnet/utils/run_preprocess.ipynb create mode 100644 example/gluon/lipnet/utils/run_preprocess_single_process.ipynb diff --git a/example/gluon/lipnet/.gitignore b/example/gluon/lipnet/.gitignore new file mode 100644 index 000000000000..9a6ee993b157 --- /dev/null +++ b/example/gluon/lipnet/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +utils/*.dat + diff --git a/example/gluon/lipnet/BeamSearch.py b/example/gluon/lipnet/BeamSearch.py new file mode 100644 index 000000000000..1b41bc0020d1 --- /dev/null +++ b/example/gluon/lipnet/BeamSearch.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Module : this module to decode using beam search +https://github.com/ThomasDelteil/HandwrittenTextRecognition_MXNet/blob/master/utils/CTCDecoder/BeamSearch.py +""" + +from __future__ import division +from __future__ import print_function +import numpy as np + +class BeamEntry: + """ + information about one single beam at specific time-step + """ + def __init__(self): + self.prTotal = 0 # blank and non-blank + self.prNonBlank = 0 # non-blank + self.prBlank = 0 # blank + self.prText = 1 # LM score + self.lmApplied = False # flag if LM was already applied to this beam + self.labeling = () # beam-labeling + +class BeamState: + """ + information about the beams at specific time-step + """ + def __init__(self): + self.entries = {} + + def norm(self): + """ + length-normalise LM score + """ + for (k, _) in self.entries.items(): + labelingLen = len(self.entries[k].labeling) + self.entries[k].prText = self.entries[k].prText ** (1.0 / (labelingLen if labelingLen else 1.0)) + + def sort(self): + """ + return beam-labelings, sorted by probability + """ + beams = [v for (_, v) in self.entries.items()] + sortedBeams = sorted(beams, reverse=True, key=lambda x: x.prTotal*x.prText) + return [x.labeling for x in sortedBeams] + +def applyLM(parentBeam, childBeam, classes, lm): + """ + calculate LM score of child beam by taking score from parent beam and bigram probability of last two chars + """ + if lm and not childBeam.lmApplied: + c1 = classes[parentBeam.labeling[-1] if parentBeam.labeling else classes.index(' ')] # first char + c2 = classes[childBeam.labeling[-1]] # second char + lmFactor = 0.01 # influence of language model + bigramProb = lm.getCharBigram(c1, c2) ** lmFactor # probability of seeing first and second char next to each other + childBeam.prText = parentBeam.prText * bigramProb # probability of char sequence + childBeam.lmApplied = True # only apply LM once per beam entry + +def addBeam(beamState, labeling): + """ + add beam if it does not yet exist + """ + if labeling not in beamState.entries: + beamState.entries[labeling] = BeamEntry() + +def ctcBeamSearch(mat, classes, lm, k, beamWidth): + """ + beam search as described by the paper of Hwang et al. and the paper of Graves et al. + """ + + blankIdx = len(classes) + maxT, maxC = mat.shape + + # initialise beam state + last = BeamState() + labeling = () + last.entries[labeling] = BeamEntry() + last.entries[labeling].prBlank = 1 + last.entries[labeling].prTotal = 1 + + # go over all time-steps + for t in range(maxT): + curr = BeamState() + + # get beam-labelings of best beams + bestLabelings = last.sort()[0:beamWidth] + + # go over best beams + for labeling in bestLabelings: + + # probability of paths ending with a non-blank + prNonBlank = 0 + # in case of non-empty beam + if labeling: + # probability of paths with repeated last char at the end + try: + prNonBlank = last.entries[labeling].prNonBlank * mat[t, labeling[-1]] + except FloatingPointError: + prNonBlank = 0 + + # probability of paths ending with a blank + prBlank = (last.entries[labeling].prTotal) * mat[t, blankIdx] + + # add beam at current time-step if needed + addBeam(curr, labeling) + + # fill in data + curr.entries[labeling].labeling = labeling + curr.entries[labeling].prNonBlank += prNonBlank + curr.entries[labeling].prBlank += prBlank + curr.entries[labeling].prTotal += prBlank + prNonBlank + curr.entries[labeling].prText = last.entries[labeling].prText # beam-labeling not changed, therefore also LM score unchanged from + curr.entries[labeling].lmApplied = True # LM already applied at previous time-step for this beam-labeling + + # extend current beam-labeling + for c in range(maxC - 1): + # add new char to current beam-labeling + newLabeling = labeling + (c,) + + # if new labeling contains duplicate char at the end, only consider paths ending with a blank + if labeling and labeling[-1] == c: + prNonBlank = mat[t, c] * last.entries[labeling].prBlank + else: + prNonBlank = mat[t, c] * last.entries[labeling].prTotal + + # add beam at current time-step if needed + addBeam(curr, newLabeling) + + # fill in data + curr.entries[newLabeling].labeling = newLabeling + curr.entries[newLabeling].prNonBlank += prNonBlank + curr.entries[newLabeling].prTotal += prNonBlank + + # apply LM + applyLM(curr.entries[labeling], curr.entries[newLabeling], classes, lm) + + # set new beam state + last = curr + + # normalise LM scores according to beam-labeling-length + last.norm() + + # sort by probability + bestLabelings = last.sort()[:k] # get most probable labeling + + output = [] + for bestLabeling in bestLabelings: + # map labels to chars + res = '' + for l in bestLabeling: + res += classes[l] + output.append(res) + return output \ No newline at end of file diff --git a/example/gluon/lipnet/README.md b/example/gluon/lipnet/README.md new file mode 100644 index 000000000000..70eda1765bd3 --- /dev/null +++ b/example/gluon/lipnet/README.md @@ -0,0 +1,194 @@ +# LipNet: End-to-End Sentence-level Lipreading + +--- + +Gluon implementation of [LipNet: End-to-End Sentence-level Lipreading](https://arxiv.org/abs/1611.01599) + +![net_structure](asset/network_structure.png) + +## Requirements +- Python 3.6.4 +- MXnet 1.3.0 +- The Required Disk Space: 35Gb +``` +pip install -r requirements.txt +``` + +--- + +## The Data +- The GRID audiovisual sentence corpus (http://spandh.dcs.shef.ac.uk/gridcorpus/) + - GRID is a large multitalker audiovisual sentence corpus to support joint computational-behavioral studies in speech perception. In brief, the corpus consists of high-quality audio and video (facial) recordings of 1000 sentences spoken by each of 34 talkers (18 male, 16 female). Sentences are of the form "put red at G9 now". The corpus, together with transcriptions, is freely available for research use. +- Video: (normal)(480 M each) + - Each movie has one sentence consist of 6 words. +- Align: word alignments(190 K each) + - One align has 6 words. Each word has start time and end time. But this tutorial needs just sentence because of using ctc-loss. + +--- + +## Prepare the Data +### (1) Download the data +- Outputs + - The Total Moives(mp4): 16GB + - The Total Aligns(text): 134MB +- Arguments + - src_path : Path for videos (default='./data/mp4s/') + - align_path : Path for aligns (default='./data/') + - n_process : num of process (default=1) + +``` +cd ./utils && python download_data.py --n_process=$(nproc) +``` + +### (2) Preprocess the Data: Extracting the mouth images from a video and save it. + +* Using Face Landmark Detection(http://dlib.net/) + +#### Preprocess (preprocess_data.py) +* If there is no landmark, it download automatically. +* Using Face Landmark Detection, It extract the mouth from a video. + +- example: + - video: ./data/mp4s/s2/bbbf7p.mpg + - align(target): ./data/align/s2/bbbf7p.align + : 'sil bin blue by f seven please sil' + + +- Video to the images (75 Frames) + +Frame 0 | Frame 1 | ... | Frame 74 | +:-------------------------:|:-------------------------:|:-------------------------:|:-------------------------: +![](asset/s2_bbbf7p_000.png) | ![](asset/s2_bbbf7p_001.png) | ... | ![](asset/s2_bbbf7p_074.png) + + - Extract the mouth from images + +Frame 0 | Frame 1 | ... | Frame 74 | +:-------------------------:|:-------------------------:|:-------------------------:|:-------------------------: +![](asset/mouth_000.png) | ![](asset/mouth_001.png) | ... | ![](asset/mouth_074.png) + +* Save the result images into tgt_path. + +---- + +### How to run + +- Arguments + - src_path : Path for videos (default='./data/mp4s/') + - tgt_path : Path for preprocessed images (default='./data/datasets/') + - n_process : num of process (default=1) + +- Outputs + - The Total Images(png): 19GB +- Elapsed time + - About 54 Hours using 1 process + - If you use the multi-processes, you can finish the number of processes faster. + - e.g) 9 hours using 6 processes + +You can run the preprocessing with just one processor, but this will take a long time (>48 hours). To use all of the available processors, use the following command: + +``` +cd ./utils && python preprocess_data.py --n_process=$(nproc) +``` + +## Output: Data Structure + +``` +The training data folder should look like : + + |--datasets + |--s1 + |--bbir7s + |--mouth_000.png + |--mouth_001.png + ... + |--bgaa8p + |--mouth_000.png + |--mouth_001.png + ... + |--s2 + ... + |--align + |--bw1d8a.align + |--bggzzs.align + ... + +``` + +--- + +## Training + +- According to [LipNet: End-to-End Sentence-level Lipreading](https://arxiv.org/abs/1611.01599), four (S1, S2, S20, S22) of the 34 subjects are used for evaluation. + The other subjects are used for training. + +- To use the multi-gpu, it is recommended to make the batch size $(num_gpus) times larger. + + - e.g) 1-gpu and 128 batch_size > 2-gpus 256 batch_size + + +- arguments + - batch_size : Define batch size (default=64) + - epochs : Define total epochs (default=100) + - image_path : Path for lip image files (default='./data/datasets/') + - align_path : Path for align files (default='./data/align/') + - dr_rate : Dropout rate(default=0.5) + - num_gpus : Num of gpus (if num_gpus is 0, then use cpu) (default=1) + - num_workers : Num of workers when generating data (default=0) + - model_path : Path of pretrained model (defalut=None) + +``` +python main.py +``` + +--- + +## Test Environment +- 72 CPU cores +- 1 GPU (NVIDIA Tesla V100 SXM2 32 GB) +- 128 Batch Size + + - It takes over 24 hours (60 epochs) to get some good results. + +--- + +## Inference + +- arguments + - batch_size : Define batch size (default=64) + - image_path : Path for lip image files (default='./data/datasets/') + - align_path : Path for align files (default='./data/align/') + - num_gpus : Num of gpus (if num_gpus is 0, then use cpu) (default=1) + - num_workers : Num of workers when generating data (default=0) + - data_type : 'train' or 'valid' (defalut='valid') + - model_path : Path of pretrained model (defalut=None) + +``` +python infer.py --model_path=$(model_path) +``` + + +``` +[Target] +['lay green with a zero again', + 'bin blue with r nine please', + 'set blue with e five again', + 'bin green by t seven soon', + 'lay red at d five now', + 'bin green in x eight now', + 'bin blue with e one now', + 'lay red at j nine now'] + ``` + + ``` +[Pred] +['lay green with s zero again', + 'bin blue with r nine please', + 'set blue with e five again', + 'bin green by t seven soon', + 'lay red at c five now', + 'bin green in x eight now', + 'bin blue with m one now', + 'lay red at j nine now'] + ``` + + diff --git a/example/gluon/lipnet/asset/mouth_000.png b/example/gluon/lipnet/asset/mouth_000.png new file mode 100644 index 0000000000000000000000000000000000000000..b318e56dfd2188f1fff9d1594e39f4e530ce00bc GIT binary patch literal 6372 zcmVZL z-PH*O34j=&Pjz-yR#pxF=l}ZO|NX!Im;Vv}`0dxvU%vkHpZ@fxfBMru12}KnfBL(> z{rkWFC?Rwkyee45pN+grKTrRhLuPkN-b#B|I5B>KaPusSUjEIN;OmhA$ ziKIkH6iJRW0K>LH+b)^{2mmHP8wLQtK}u|`>~H(+dfVHj{F0Rzt;Iv%K0Q4C-5>r# zRrcOKeRzoD?f}i3v{|4*0t7)DRc2MOQ02^;e`i%zm7x{@Ac6oXl0Z7P)2Zvs5JW{t z0+3askYHlo2S7p;`QgKdj||w}$GGh>TC{o1&_o0Y61MH*o>679NU4q&0oyg7xNO=- zu*l;r3`nL7gEG*b#V`Bqx?g2>`rJbw&mbN?K0bapTQFhJBA_x&7YqU*V4J{wN4(x_ zrjbSgCfnA4wR<2<6xl@dNLN;wWvWn>Iez__1qkn>g05GF@TlTi(#M}ssp_l}+bTzmm=65?F$`jb_2tG#l8 z6q=z5DFOuj0(-;cZ5SaHVHEu9`rx|msBDt z0SUNWqRIY${`maT$kXHL@e_wJzzbOP00GwAiDDMZ@`Ny|q`5t&Az_Rxr|E5Ltu=xP z&&{8kNWduEfe8>Olv!W_MTCeHs4!S28Dx+$RwgA}uqly8Qkii&VkS@^f+C2_I!;(h zsQckNP9lE178G14P@Z`TRx1*~&i48H%fDQ{{`~2~rcyASsETG6EHB6$S@yo6bD<T_Q=Vkvg6~wdM8*nQEwFh( zXT^X7Apry0+WGs_J7xSi{z+E3st^ap6s`3ZT{YGxkrJ)< zxo=?#hxnVpx+CuG8~@Vc|JvmPznHquXty{6_{TpK9sl(5-y-Xeui5hY``f?%bjyEu zuD|_V|DC?=w z5+mABWz^|u^nuA$5v{dOv-Wn{C?=84VSAXuZ5~;PLRD2JX-PG95~ql*_jB~B8rR!C zc0IJY5*2gis>0mnvJ0L}vo5jqQv)g!Qs!vuloAAz1UYvwsbr;5enaM8y=D%Q-isYc znYNLc^L3FS1*1_e=P1B)sYiOGRt-7XWM<$gDmC*{nOYhRq5xFoQbac3%)~qJ{yU4G z839MdQ&aNXuUTr}>mj?%5BlC4?C}5g6hx^`?F5-TIBGF6MRD^;15&NK?s3q8;ZG#3WAStVPKI?4!o0 zRB6gYyk)3&n0lkw)*7KP27m=_JETQtiWUvVe%qhEJ-vK?uD!-SY8YuzlyI6Vf7W`g zkqAV@#){TriwHuo&Q48aPKQE)z?9vzjkHxc^1jx;0I>;pLlYO^O}N#r7#T@L60M1N2o*>rb*)BK>{9?m8oO{n_@%fj@ReQ)2~m@ zU!QKTS0{2$6bLmnQGOmqValnnBuA3mzQ^g|)F1nHKD90o5caCeZJ%0KF(n8{a55=8 zipsr303{)TRs9^8vhG&RdZplU%`8}DAf(u>q*kz4STx~P@mnp&xDM=zv1;*miiB^O z@ts71LRrqt%*yEkX0q6OCv>~+m)ECXKY#o7%eVcq(_nFW5QGLxWZ||<2f$$fo;JjD zZRhQDIc?|DcJ6)K+7|W3NiM^-1nIt514xjBY>-(w-5wt*(bFajRn@HN@3z`J9Fs^5 z77otZMjXpRk;%C6bfsT%e|5W|(Xm-v4%I5d)>aR3opims{_b)2Tmxd^|mz;59jc1fK^RvX^;a4vi#%U1rBNlFUCvx#%PFK_0vo= zhnsazTt^9C)6eq?S!Q_@uwP&IFJHcV`TWcEvWrdR$f1bsbZVh}A60oepM#ZSA7d{s zN5fTsTWE|?d-8him-ceoA08hbK0R&^CkljIS*~dkP?k6ac+GO<&ECT*p%2!Y$zg$! zq$q0&W=RIiRUj4Zu4n-l?vHU%DEikCUxr^Ex5hngP}N3D#HtTxxq(^P&#hRtftR)# zW8_O#U7o(b{_^X$=hs^m6@(4v!|C+9zxny`H409XhrG0;2mqXzHEUE( zh2=`8K^(=vaH?sP_sndJz8yMinN91U&uobp*V`zYa`w~dbUL3tet7ulMcWPF1Z?5;ODBoSHUYm^KBP}lWUoiCB zo8M)_;v1%BJd#XmXI!0^jKNk`0CfI&tjy_T>8?u)H6yi;@%;Sq?c4XKr`J6zEr6&| zt>xIq^|ie`-J(Vg%dZ}T%I;EEe~17{_Or<7jpTmWpT|wk-dd2{b6juxEyrFDpB~PS z=c!U{dSRcnQr7#zZ~@}(qpgEE6T!W}=i$5kCRnkMbw9@Ifx&&$ZLbn-maDweu|DmP z4@;nj6tgNsRjoeQzTfui>*e)wy_ljpsUk1e+y3&ZXI=LF`(D*q3A@Zu2C1XS^jmPrxm`0-`#x^B z+ZcmHgj#Q)6di2ExbFM)wvX$$j`4hXQ3x^!wi;uf43p`I)}mLYESL9}+deA5DiAHC zQPmFV=U!QlKb_ThiY=EvhS0QPQFfsUCFsud6B?$EbbeTvwvVKL$Kru3FG+IX z6SH0^aulxV9oR*?6uEIWGvBO5{OB&&Q)-PdM%L(CeERg*dd~vbz;3kGH&8E^*V^;> z+f%gGPi=Z@W86j+Nt=j5xH4%3;}q@T!>54G+v(wS-U2t&*vI91`TqSST4U$q#|Ip4 z_tG8YJX1gN74hyac&|j?@zipNkLw3yC51QUgNX!&J<16T0`$&ooNDohNWCL4EP*ve zY1Ov%-dgYDx@S7bZnqA!-nVTVnYVFM=zKc0-sD?VGD0EOOd2i6*snL(;povDqeYzh z)&m`mXxIH(91(45q=R;*-!>WNJ-xs2%?~R-Zsz@y_y5!A0m?{R8T zUz&f%@u8h!f8buJsp4!NPMy$tH1&NSx7$8O=AKS(Qa_y@w~sV)ugm2&u7i-&vwRJY z(}$6Dt2AKIBwO1~Tkn1BJB)4HWL{ohFME;fWN)!W^cKAltP$8wtw40QEp!sw&8*uP zX*9Hu=x79*Zt2yllNMdLKrpyESLI|Svske9+d!pAx}CNUZAN^W%Qbz?*?b~7xm%%*c3?Iw(ITX>T*OIdtHG64Et8K zXWq(XU&d(N&f0JL^eUoIxBVQgMVuOYYg>!Z>^009k&G5+#TL~toaQ7bFyRWgG+U7K z2WIhjbXV^^x!f!em#+a|C@gfwmg%OdAUk4%vPXqIB>cH24Jhf$v1?q;4(rxtN)(fb@}Fi;8i zh(<^5p^>!Pq*_k?KJANy--il`zfZkdY+iHtOFoFu|!ppb~+pUSLiE9IdN^YGDiH+OI zS+-?J+@z3tX#Uh;FtfFQNRhrT^1dgsXi<^imOzlfBptA#?kTQt8-32zB;e#&)>%WaHU^8`GGFpgsn z0Au9HWpzcU#Tp0#QI$4C8BnOTmMvWahhL(1uJk=vX=n`{-RFo3lMcxUMH8D#Gz(#g zih$TM%Qfd{>Tq%HeqH74|sM_9`!vq4S= zwW${*{K#2<@e&DYR9^Ry)%~MR`vHsx$ha8g{x77kkg#+KB3YhA53H6^Pm7b5jmKLCG0K+WYHufGG^pO?DRp#aN8p+#G>4(q)v969kG zin4E&0L5rLeHr7QzwQ0(-*5RLMzR1mnS=^k?_g1m2-|c?%d9UZoHilE7EWZ9g?4I5 z6{x5jnIkJx(`HH8YeXVF80_b!P&7C^B25RS@U}Ix%sNa3+SO*%j%f)1`+fz{&S6)g zs=S62je;$TtYwLjRMlR$5sb$VA0Hn+K1=roTgf67kZK^yXv%9<{WmBm=X3DsrrL126|WMj3|; zvsq=7vvQzz^M$r)8NM=t9;u*@J9MMRWP0tp6E6;y*&x!Qj^8U;kXFNA3`Tf`bM!ltR32au$BK3ASMM8R8f1@ZRE(DgLZ2&ZCV>Y zrZX6n<_9mCYSsjL+Etk~=2p&eLb!;;4;g;RW0*&3UV~p{PPu5jIUjRq;m|T9r8xxP z_sBZl54?3eS>=&A#_Y4yL|E(5dv7AcC(!XcDAdwi_Xt^`SN=G zIc`5iwg9El>SoW3E~)OszF3w8t)yI2!`S619Cms~?LrD!DuEpLQ>4SHlTa!OkSU1w zG$X9m?{u7~$qYtN3K_NvB$(9OD6)Zfqcm$9vM>|msIu6%-kQV|>V!S=`WG4&E9wV3 zh^j)C)41;YFVEk;*VDeawXSzYERJsYJeElRnaW?C@u`lh4mq=oYMS;u$oB!5i4jvG zG>6L|z4rwcS*{71)i^u`NJFzXK0y?Ubgh2HvEZ52yvATP#+Yip_kP-Xh;Rm@0dyU~ znt(q{!6W$djI}Hea8NIM?O(tDito3dwuf_{9jBF&f6Ug$qIoc{-d|pyzdil>>%WfgPe1?R@nbL|j^6L&36mBf zE$@-CEr3*U&RlXCs?#$N1T=6Cf&1N%!2Drwx*A{{f4ItF2$ZRUruBjQD9Ne;+Z^Ch zxgg~_wyTRpfYY3HrK{}BL{8 z+b{oi`SQ#E`SSHY|Ma_ukMZH-cJ8M4nd)H-u*3?I0+~f;H5VmQVu58A8aPryH|1(| z7o@n(LFRdIXuu(m2sva5M=`_@6;{-2M$|lpf?7@hwbndvLk%6wn}HC;T*Ti zetFH?_4VcX^DkeXpa1yNPv?xH(Sni&2JYpzVPi(=UVEd8YhHMoYlT3?yP({pcfwMGi*llaQA91IT z97=LYhkJhp?;a3v6+KkL<2J4@`|Wj%%k_5ox6hxye*OCM&mS7H$wo|M*axzzxY&gx zR~v+v5(x-b|_ALJ8fHw5Cxj^tVhgGJp8QR25Wr}U8wZ9jmyjJ^?BT0Gq3yDzkMH1 z&(GKE^`v5B&XRqmox7Z{8S{?IB8Zi^mrY@TU7Aag8$Gp*VviVChVT)De^EY1Bd_wt9+cvT9qq1nWh~9c{EtXsZr6~vii)g9?zHiq`v9#st zt~PGt^1Q!(&+%Hm+IhXX=J+t|X$e)4!I==}Q1NQ4Ew2Q}w$4<5a@Cx;J7g9NS}qr< znb?XXG>0eZ@VwvjH}2E1OD-LD`m*=3sF(7lMUy}t2oJXB4Mp0$3NE>|*n@60T4)ks%G m>pfc2njO@^Sy;_%gZ}>(3pcvVTzy&q0000)jTr>m;6a*WYO?_=L~k0jaJ zLJB~Rlv>%hec$%Iwbo9jEkdFMSK3uIw%hAIHj`2VIRLwgf~le0w;20*Z66*lmxog{ zAjpse5_A9{!ew3zfEFbhqJm+&stQ#u6Cf#rNbQ9Z%eF{T8O`9W>XxHzh+8sF@3>3W z?>@ZGzyHI}PhWF%2DK=HMrCFgAPjh3fQfk>zZ*1Q6kLW)mu({yttlEn+awGV2uE{} z0Nq5poHkrWmWOGCC@Clb&ICZa!U^OOJgv*=bXt~0VTIIm+o=jUspXW=ZA;$ z`ErUSB$B3R+5iO?ECZhDJeNhlcHOtTP&L4{#oid-{}T zBF{$+AM39p+jyK9A%cx;-F)8m&o8e(q+46wujkr!$Zc7cwSgE&LWJSsu?ibDQRRD9 zf1hta83h(4Ng>HV!XPDqz_F#40+g#@DXAu7Uqkx%H_S_7blGv7oJ`nYMsZS%zjE|BHqGSta4I=3}Gh2{1g19LvCbm$oUly+WrzQF+sv6TKP~~J9NeG~48WSMr+a+o^$pTQU(y(0`SgN%pnn)pOC&)cJ zW}XKfGEpW%Mvk66YuH6I*q{(agxmtLAQp0=MLQ^MNXczBKtr@ikRU|bAY^l|kyQ#) z99eTW4I^xT_b`!!y*g~!WjG;GoCK;iEzdxDLmXM$rGeBlp6oRVRdu-CwL=wo| zgd`YQb`cR_qF7X@g#=M{k(7cYT>_7M7w+Ymv!>6~2UtzuJdZX?f*4iz`aUB_XoicN zXORR%6Xl{r60X+EI|1XD>-N*v=RZ8qY_6lJfcMZ(9QD^ONsgEqXiX@`bzLK3Oe9dp z)2O4wycH7iXdWab$_GfA{{uluEg~WUvxeSZCmv;+BqdFB3BdQlN*)_&R~049nm@B3 z6Xe8we9Hj4CPw**>cpM6~9r5$%`f zmp^^^{5S6(F3Tv5OKLn<-Ob(Jo>3s6Kt|fvmBHV zm+gauEW?$Czrk30G$Lq@){jptf z=4q_4PzqN5)v`5XPe5b>M}YxrjH7ZK_-m#bLlFo`9yzMl$T70_J~GQiFen8L$_|p| z965smk74Dg=QCSp!#dh*xk|pzcRccMCTS?%;3y}~_=^F~Z)04|k{uIg6{?{;7zqCS zaFKES-MUo$bp2iH&%g8M9|%E@b-i%7>U)GYV>BCBBeM?!jA+Y3Y2U{_XFYphB7MR< z$N99L8oA%L{mb_H{M>Kb-h0n63w5;CLP55OLPZM|YTNr2>$}JE<-Dr)b!2AMFed#` z&_t>8+;Re(y}HZX^S*!BbBJ=Lzx9xbDef1Uc2Ty25S2hx&Y86Nz(})UVh@^njaYHI zHMGh)7;QnUyK;dDy#?$V35>;CZZeLF8!1x3Zk9DR&RN+OF|A{5d*YxWqaR&zLg z+v&Ff_ATLkZ+v-_f7(VFL@`7aZJ-qw832X=;L~}L@pWmJJZ;zN@%mfmSvu7s?B7?y z?`sxk?-}=wYbR@GCWcX&2C&4sEX#hYr!P;R{`JR~uP@xoRZEHDfX7IZ3J#7?+C?Lz zbdDTe;Ooow>h-*|-}rpF#L^aNC^O6HL1W(1#7!g$1&i61#-nA>gdx`fyeN7Ova{OF zl2WuiGA9JHCdyQMyJyW9vj8WY!cJT>U-v#POf!JDLikp^c;I-?5n+O<8kLo^2-p17 zswLK7eE#w0pML!L<xgIVP)m*J}0%eJd&x%u(kyUXL{ z^l)y=>QK)uhx4;81Y@O$osTUo32W`*ur!=)0MD^?(Cxn)vx`!U;q7&KY#w^OYX9f zoC|Uji?S%`@^~TLx4kl*BgjS>blS!2WsFcBF0Ug?0rWhaopmemJ~-w}-8 z6N%*a2>9(!-ywywJcgl{>&xd)pFe+ozFxN)i(w#_b$$Hs;o;rG;Mn*5;qmc@zx$z3 zpML!5_4(`d`I%BXom_T~ag?aR*DLYp^r#kb zd3Y3sK5n;Le|qYDKc!#|`*~bnZ~eCImk*cIWrUFi_%KQ0@UbAjR7w zmNQEzPl}brN!D8?TXJK?+rcdsfHny)EalU+?=QFNm(#-2r~wRVGBtDLpccCdO)d>0 zQJXSZwzIGgZ@aHA{ps`V=g+TSK3_AdaC-OQ!)aY_(sOwsw8qK{&+$60BRfwx>wK;Y zTs*h7EF5dyBD`<6*WBqWEkdFCzHNQm`iI{irzS&RsNntl z)SX}EtXiNWh!mBGkl3L0tG=zWZ_ zM7XMYjXrL#w_mAF-VT@stLo?_ji+fe%uk8P!(R#8JCR+K`==z70?JrDk>o=AF2 zoe6H|RBf=$IL%&pFYj~oVv!byvGuW!%5b>g48f4^FPKys!F#ZL2x3(u$EcC8L+#;w zl0HP+w|%=gTEpbr0GD&**0-nUr+@qD$DTcNOlBt{)=*`tWP6E4ZE2TFjWM>{^Y!Yi zWzplqYFZI!e)sWVku%+jn6vtQzInU;l9jt(_etpchwhKHD1il#B>)uWJyfC79qeqk zj4flR1t#J!T{v_JOm>-&L=cswva7}zqxZ@T;Qhm;aoukF?a}+*N1t_q23#G$dTL8+ z#mip(b-R`))nP@P+CreIHR=83{dziWU+ekl>-O3PP`#5dTNSsh->&=la%oFATa_Zh zta24BLJ`m*xZuu@fBV6ltS6GtTQ|VbWZCF%TIu((EW74m07tqHRGVP*t|8tTkC`2F z#PD5E&M|uLndz#K-aS4(UfOosUvIhZoj8%~J;xXYXLZC_&!>m?4_xBfZ*5)9=ku-a zBtJYnE^X%#LX)@2oSl*?@ZOUPMX<`J9p z?|*pNf8Os&+C`$7f+sUuz@*7oWD%X(7&o=6*Oz3+Y2_;Hi9sh&iI#ASV|t*dEI2dA z7$Y;`3i08*tZj{2`*QkHFDXTw0FJ~Mx%b>tJwW&J6DFApN&F}3dqpH8t}g5mNQV;=uno0=jh&Pk>+LVtGf>TM{QFqF!w zBP@PnRoL(s*PHTOcEO;^809(|z0 zHOAhb#;81@QRK?LtPiIT`@Zk{-YZWEe7WV+Oxu=3Y98jACKJ0ww0s+x?mG9ZTt3ESjo`J`FXgXw`i;18TLopG5@=PCDilaC z4*?tzYSLkJh-O9_92jlEg|TFVx4IT`D0#ABfkJ=^b2c!O^i&9e5M-eS;ROIKf`M45 zg=$E-R1_(MV!_hkF~+P#>7oy@c?vdVo3eBle5v@FIxo$P6T?7y06YYfNsgzTMKUyl zB~kL!dCTRoJ4eb~Pr+#Ww)d^qKpQ}sSvoY$!vIm`L>fk<#9|Rpt5_5()R3xT5oN)$ z1s>96Et;CNJww7ex)AT-tIEvD8qeKCva-^dnSd&5miEa?x4#ZM-b6rO%)2!KZ}^olM-q%8Mxet_uP*2vAeDJgAO@<_LsD zRl>IO7``*@%sxiiqjFkMt#Y_rE~93+F}GlfLv#@iuv`aWQW50?lh274@ZIrX18xMO z9pXEkGz7tueQAcT4ffqr6GR9BJey2n%cE!vY5%r{q^hZ(@+2Y^vmCWTzW*bX^F|%JJKvbA;n#e zHRf`@nNxtOoGou_#@^9>Fz z>MqAlLC5AaHX+ODvUNXD9;0WDs(|6F)?!)KDkp}UTYVGICPOvPurRfL+fHIs-=4qx za{Kc6Q~mh-cfZLG9}$yOi|H9UqKd%B}z=uq{#N21;MuTt1fj&&6q|TBn8;WdqJM?d5X-_vNG|BvufnX95ypkiRFB1 zZLv)WM7vi9OaYbD+kFFuQ|^&_U7xluzwA#>*ZNv}Vv6RxQ-DCazA;9aCK>APsqps= zv1(+MD-TyP(%7n|Lz&VvxtP=VLuf)$SW3WjI5{nwU>)T4yoV$47*qY5?N~Lvew^AX z-^ODc+}8Y)S(w&edt~;N%Vk}fg)~+`r>_u8h} zIx}Tq8mNJEiQ?fj%exWFcZ=P3`|cutCw+vXhth_Hw_l&L#AfN3#1 zY#TJ@|HV6CVT%AUwU+xR&4XB(Y4OH^@*oBP+9)T1^J!gMpbW99Do0a|A}~G3)9o_1 zXliFp=FaWaPoMnywdIJam2rwzz1Jv-6*^uzjX@Do*yNER^XTC4-ou5mSjYc>u&Nfh z4lR;nIs+k0r_}7anzVHhhzOF5!%Hd6R08j2S8;yXKp@LYRR!uGMkLddusr4}qN>U& zsO&KYa$MHa`_pN)S);W!5qJBpV9f$t^Tw)DhwM=vJ-2J^+cJ-zy)Nt4Z@up%&dV8h zHEBYW>F9i8Tw3x;T+lP@YQ#yv%>5x>2I&AE8NeX2sD2Jj& zv+J;#`(QecKFmW#;kWPV9P^uTH{ni|+S38%QONYZ@4ffTaXOtY>v_INqGm~|D!akg zaE_5yCveIxII@Sc?JxfNwLbs5pMQ+~m(gEq>;h$ku}0JMMm1YV3)ZNiW0(M-$)Qf| za)$1}x<1Hz8FK?v!*;nm*@=?lFKV1!mP~!lku6M`ebUg>pqWGiRp*tZ&%R zGUk_WCQ+N2nx4ujG9GXY23?c#bN~YL@EQxi-ElrAgbCMuH&PZ*6PX#db5fk6oLS}Z zMlfg^M-3jbxyQ&fVp$*0mxa;5HQIDWrxG$lNw$}t|BXFeH78Y6r|rJejtOy5V~jpV zW)Y~WZQC+4!Xde8j84LC)4N0*9_d*+-oEtZ@3#9;)W|yS=pZqDM`Y=`_p1YwWaS$; znwD7}!g7hO8SR;7Z^O;ZtgMk~S5D9Bbm8ZYIEP) z+ma;Nah!X^$*k(j%+6vLq_8lGMu9Y$^a1(?{iD7`pP+ZjB!LEsBtT-ZJ3G^ts>~Da zdWfu^B~>=l*4A}9c}|3fhkGRd>wo+2|M|z?{-|7QI2KmOhS`rF_BkMF<#9(n(Z zuipLjuRi?EZ~nud|MJ8A`5bm>R!v=JZL{W_`*aly03=t@wr-F9y-GFfdfi?x*X!l7 z?>j|;@4x==-FM$Tzr1|>>8J0XKK|hkf2i z+U}Ku1`L`(z_zv&f>hadkYut13;<({V9Z$`K79S|-Mb%u{9&7O+qP}n_c=={$+l%% z02mPwV;Le{0sufZ`m3a>swzn}rP8)-+jX0B5+n&zRWC0u&(F^vKYskjKm61G`R(6* ze184=-~H}y{`xn+{`GewM^)LlZd+BsiexOOs`~vOfB2t&`@jG3_fOj;*8g~Ue0BRBp`ZWzFh{3gZ(i!3ELot2`W4pF~0fm?RUTY zvw!;C-(U92yymuJ+r3ZCQqoqrssItn5WzkiS<>c$!TQ&aP1-y!Q(GmHBuIeQm+dn@ zU8UETZNK=dcfVNgwhTUfe13X(IiJsf<=Ur8nkp7GuP?8!fBz4E{D9?JNQGx)nHzh1c$8Os&X@eDEa2P0iFD>hO ze>Z;l&9@^%l2o(iKKH4T)iJ}~>;6_C7!JT9*{MgX`ps2y&e~O*=3bUM9?7sx<8rxd zwxq|0hxcE-&+}=FT+++37$h-g%_>Q~zg3d_{{2^f^;eq8K|I{8AWPb|9UvH)%ovOy zGb0g}OiC6R0LYNsM`p{%<#Z4oQ5O+}Yy(FC9$^ZM9jaJfcCGL4PRscE@!h}w)pt+V z-$k-qPp{8cf4C1?B0&T~uwc`cVKBunAf=K@dsS)QtM>A4$QhXqi zVl3-A&gU3m*0DMYD$jjNF$PGOr}h5N|NQ+o-~4jlY}K;8tam5oVy1y*9XWzbf@F{( z2qZ$~8fctS1ng>*I^qieB*+ksCj^*b0`7PML=M^0*oDj`b5Q<^U;X;iwy9>RW}}al zL4cn1hTpJ{KqGtuh1(%m$3ejaf|)s%g@o*7T~z>(5i7wdMs!v*JRED6aa^l{4Ccte zSiPVIt+{&}BXi_fmh}{4WF(0ogTWgv!GP?B)CO=gL>;SFXIy8@?Wjx=Hc2oTEXv@p zy*plt;8>TtbBtep_2D;v`4=BQew?cF-CZt2bU}3G4JdK@DNDx05m`13PFQeR)deAQ zEX%Sil59n$DBFaWksOxo3}#qU763_AkOIg-=8?akRaM>yIPXs9yVKqMayp-uHFNZy z2N9s90@$krI%W|6DzT2u>ceh~I?n&72#6rSh(u6T!@$OP5q7W`ckAi8tlzxzsYZeN&OE4OF zB8fa|oO|s95rJ_UcXy}LX&EDeh(L}Yx`vo>ppX>4MNK37@xN=g-=5NK$soh9>>xV| zBS>=~xJAAK&)a<6=VRpky1v#W?(TV7?(fb)@CZmYHb%rjCT`_ek_i#$E$d|x2n1tU zmN7&rXdZP(}2%6-GjEHpR$Z^8Du44=mw2q_6U_cU) zXj)Kir~&|o-*`jk9)A{Sx8EE4`}jedHbi<%aL>5PKYsrF&HeJ;akrfA_7RdI)*LjX zia-GD_t^~u%|V-pk!UdJ!eIggM=-}Ytpr^Kldz_o1rPM=ZW zPHyz)=jW}y8kxxDepv_ON`(zWRfr&i0F3HngUPo;x}D1LhXX@n$#Gg^4A|(%v)TM5 z$fWEDMv|S;ovDOKkr4!w$y`T-G9t4gBET2~1=@JS2j(&|2?80mOiN9dl5B#-H%{O$ z`AE>XU5wm3wigR`Oi%{Kp1h1Z%=-Oy_toX@-<`ig-Z?*KaKuEow38v^v=UHXE{UK- z5vGxFiD5}oo3)cTpOzTOK~OF^EklMnpCY(4dnO_}@{@?*Ub9dNImlF|gB8$7=DLuq zq(iCNp`bD%f(Xoz0Foov1urmI_AH?yvrFRvvg+dj;2zNVg#Zop=C96_)F}tX#oNR!*oMGO*k0m^U9HB9j70i32gq? zj1Xjhf(_S!6|z8g+Jjn_F_IA&BbaHp!pFBU&@dQan3Rn`I1Y9|GR9!~I?EIoz9k=S zeG!|YpaGdgwk?eS3##l$D11u{d4vy5w z5QzvA!Vw>WOC9=-h>Tccou0iz-5IJ(Gj9E$kNVKZ#}I)qBaqYCAwb6B(* z;Vy5TxSjoktO4MX+{y1xcc+NoK7V@t0srOUtK_^_kvmS4WMSHNMCZ@X0-1Bd zGV^>oold8H-{KK{$= z^W)usF3uVw#$ZJ9U-y<@4xu}hn^to=t>^Q(TkDdJX8)#)ko;2bqq_TEZ@cMdxAK$EaSQC0q@8o>rZ$|@>(FA2z335i9PO&cA zwr#4NxJE-dTp(kK$c%-eZ~&UQkh8Ho;PB$+glK@2%8o<4ZBy71!IqZ{b|=_ zd)_avkIvJ(u~Z~&$=*9C4fIFWb}8 z(|+CdZTFGokz>Tj$RrWLAZt!l4aXX5GMGz22INwyWhmWbq706VCfp)PRmt9CbwmQU zw)xg(bh$Hc4E~Uxex5+S{q&$k%Qat-2>$c^IgA;nQS;C5-aq8F>%wvW73Z2}ZH6Ux z1JsCfQ=x``=FAb1 z5kU@u@SK3hUEDw3-#whyF}#H7ib+B~gxTKfWh9(A06;)6N-h<+FUxScsz_Rx&5ccm zOm$}=l;it>X#V=(Wh+w36$aJb%MS`%y09{iqzT)NOdcI%Qbv-?t z@-%qQjIipEU~Kb{D*)y=3S~D2a!bs|x%y2nR+rK4&_kkB&EQ*#Vg}go?@q&zvwO`; zfWRMdN|jFIynnuD{le?JIH&iaqfO|1?xKw#49`|$Lhw`xF1jS`6H_*&dVPKQ{PF2e zfBe(_GF8e-0F6H^Gg8c^R~aVMC=0-*uBg|#e!4E}a(A~a=f&UTx~`cSTqCBM1(dd$ z5m>!~lF3j6S&oE{8X&Oc@FP@6TeSOvqMZv(m&7#52um10XBM4P1e%EML=qS2wW7+bebxp^QE3Ykk|9TC+l_B_HnZA08j??#?mtt!b52Y&N1X##jhME8T1#pxItPZt#NDC83*o z91wMD^UMAH%j4ty z{lj@ZpE%N_OeQ9)1y4rW%2ZVaX}fiwmWzZht)dNskz>|o7!)CqhNfA!$P!RI;;T8Q zW$)A)oQ2D_>+aBH-W0O88ga13=Fo(yZfr$!&dYWG{PgtG$4?)B`045MlP1^YbocP^ zaQ}FJcR$7$9t_H!bKmz(H8rbh+SP2oxm2ITSb|A=?zO+v)8%=4eY$@9)5r7u>EYeO z{rkuB{W?YdMH)ntasUXF%SwzLHrU59})K2Zz zW?c3u5C&y5$Nf?jS{NhfNJe0H8Ir3suiN$M`Q`J|)6?^djK_B$?oJQ)clUSa$MgAa zUDt?6g{^jQt0rNqRI)n!X=FqmAv}FMe7)2AQ*^y%Zr&*#VU*S~oG z{=?&Pmt#$4N|rQ6x?B7=O`Hwq?O)KHgzb*XV1NeGE>k{~I*d3-7kE=}00$;)krR9+ zUyRq{LyYMF^Ds&P2HJE>1}Kn5$#c%FYHr*9dcC|{pUeBhyYsi-{^I_8f44rYV;Nl7 zf~G`E)`%)eEF5a`XdD4hGJqI}pjeL${Q2p5o7b1? z>*H5<_mAg0oN`18bIxD`g#lPlYY!w5ZD`0Ul-oUMZlH*UZv!F~Bi;A2C(?zHisdcD-I5zJI)bxPSQS@vHNCTBA=q=A3oe zTAXVyr?R^`0PQ7pmyHR?CN&b%)kKLYX09W~!`+7|u)-~@=ckkXkyuUl2meYcy$>|NoDS_1Lxzojchd`BJL@(L2 z8PP)rNN_A!0R~1O1{*TZvEIeB2&SPq_icx~E(^(hFPH*VWF`;hg)deyOp6i8$mB9IlaYZX4nr*%863Tu&>Pb(~M@X&KG!_qlJ|hEira$DI2<=bZbT zb5_ZaO7l=E+8+rv>y@B@gW(nqu`>>zs#Sd24i6GzjZLrD$=jgHEJh{c(D-3Uy+I{) z3qU!#eEFz)p@W4XWdP$Xgnk~MKg;75&b`knk&zavWEN?qZT7xur>49UmqgjsJ#`D? zvb43sF(OAOBN-lqZ^EbhI4}dfT1$eiW#Hi}r!Y z4KiQaoqzKn5D`{&k=SR=NsUfq1>N&7k^%&%ar5!*Jm zZG%(xb=$UmYFF`?zi`m?#p7~!RT&5hwu>@_>t2gT;NXqSNG`I)!$i!?S!KJbMSz}| zI#8*%K=LggA*p?T9X`zwWpmC{#K_Tlmkm>ZlLz!0kS46jadG}M^V5~T9FONwL`j-* z)G=lh5ta$QL3@kB0~meM)i2bPY|nDprJ6M-T<}hBs+U%c^I&c^TMLVT7GyHpAflbf z%#DactxvY4+Tl(Q*h!)x75#KoPlD(OxSyK}goB2sr7A`QW6r671Qr+^Z;G!E{QrXR z5fKtvmNKr}+|M`#GZKj2fPPa|1;V#3p10uu-}JTqNdGM0dDG+ju-el)<^*sIEF38m zdC1Ujb766a1_{0``Y7iIQL>Dl_-Xe4Z}R3AO4VkZ$-_A27YnAUh3l4tF?wK%&EK16 zI?zZ(nsic*sqwO}KW+Q-9Ou&(cr}NOro3s}PEszz3`Ih)t8S|OVWEYNYLe|03>!Lz z0jYqNo;317wP7U@WHQ1#C9@)87-4NdwxHaM#$~s|b*ln)HvUJD`nQ06<3?qd)EP>W zNFX#%#uzi+rfNERZ|v@f zju@cf#DTk{qPfV?qV*q11`nspq$3c)Oe#S13)_fj+gJ!7(IdOOJ&semG56ay2H-j8 zUUirk5s@LOtg0>nf-y437z0oom!Y@F+S_BmWN=1ovApEnbJeFkezA{Jg9lNjL)_3j6q=}hH_HnZ6-gp~k{ zGN9Y2QZh1{RLW*6uR!6AnpnnObxiJ(MNd@$v!uDKd0FIXLZ}RHfPs2L6 z868oIm+O9Arv`3Pvb%|UYaMO}*%KaYa6wOKbmJNPvU5$)`ZzN{28PvYn@R#1$vEbu zf<)3S4Ms5IsFpqMXPbeL5FG28naBn&{ww?8APgO1NBFcL4!B{?L8*waJ*!UZI>tiA z0GtNRLL3_h1iK!WnKNTA?vXq5^L~BV>q9abHruyH98+x(!9WkK*oZjtRiYzrP@D)x z9V<1E-Lqv6#H3^kf&_K>ubQ@^mkN_iDwBM3|J5R%4PlJPjCDjN60)|bV{P5=z2V_e zoY0s+9vzHta&X=B-eoMA1K<#9mxQSqFS;O{-!C=c6){ZditLnq&p|k zoSsQ$Iz%H|WgrB-%^}~aG7*f-wn25H?XxVgHMxbnKb^+lq=u+YiSPh0qtj(4kO;Ty zWfpkB_>A?3i@!bRIx-On90?Xgv?-eGna&_;vq)y(=VKR^gaBGMRGpAIdHJ41IT-Zg^_8)2q}Cp zrs(+Zigvt-qy`}(Z}ZP=Iku;5T215)5ZvYlF6)3f#d%#3A;w#M6vImi%uz7}>O zB?aR!i=t^+TRv#^d(7@vO~t-Xt4Q9RPLSpL{In8Bn`!YL~TdP_pEj_3-epu4^9^#+89O#2k~N_nI%)=O11_{^|EG`pa+g{rv~x80c*LYfYlB%spL`u-d*oMiE;k zz=4cvO=eczeA0d;47RHLr9$6q;O6QinIrQylho}VfZ$C*HxSJ%;G>wr6~RDcP=Ktm z2ptobb_pd)urVU<*VB-*)b7_{J7zq9!e2NSbju>Z(!5-rpFjUNo}d2Z1%i z?@#Gv@J2*wQ$7;vfT)8y^sON7pvVZhZEEP|Rn$CWf!>~D?*Ix$k#rbxh*dgR1A20o z)k%4Izxw6f3=Jvw0CC zgxQlS$HcS69-NCH*^mdD-L<3jJ|j3{#BC0($=WgXPx6@PIWj~(Zd@>2YMX_@kPEPm zVgfL1Pt{&6MOG+IW8AOH=v9FkDVmC2HkIDyi z>k!;ft`<#X?KS&Oz8-UM4`1`R?Mo_V+=MMl>U(eYeP6h|JKw?12+C}Ejxjdv$ukh; z8F6$zQndu8+S7B)7kvJK&p+PHYuIxNh-I9%%VgDfH(E-sf$SUV7_v*YW(0clzgyJi z4(+2JdLha@?mKgHYO++NIAGjm92w_RwLMCTc26L+TppNJ*KO-tB-?ST znuE^BzFV$`ujW?!?pfF_`+nKyWef1@5AV*)a_y-QC?iz&D(8Jm;(53uOALwa&8`R3ao%k$axhy7ME;O+^ZumGl z9MHHP45WYnPm{>;OuxSe7=>0)V!_&0T&Msh?%JKzzAx9 zV=G^%P6<9gY<*Cb-#E_E0%S0MjCEJAq56@B>HQN|vI0(MUvv+#etHGPMmT}$%~O#& z*AI$$Zf+l9sb8W|d1sP&Vdld5Q9E+QC;@t&bqw|_=aSj?=pfGqyS>HwVepZ^4v3Di zyox+D*ssOc7KOC5)_X=3|WZejqat!^I&JYzl z5Se7sGREYPL>WfDR3gzE_Z9b7RX|_Y}K4@%09yo zfj|5M=Vq#7gg6 z@d{y9jvmk>WeOkC3#(J9eCfhbBL$#Z*d23r6HE6B!o0f)XJ|zNm~a6@SZnfc0c_;% z63VO`iffGWF#t2ywO$Ht#hRbBAK%=gJcmOw0}$&0r2Hw%p2&PhbAuDagn#?yc`y|kBi&*+LwMI_lz?d2-c`ol|54qJnFxW&PjQ*P4#4<<^E<8`yVr9YKc*j? zON@0GJ_u2nfY6VbS!Z%zUs3fU;fB5!%E_R}U(d%@;w~gm4zhSxopq3TBiEikWJaG9 zYLdY)Y_lcg#Fi8`_?AfwBBq*&kwy5+daH5E=M4FV_tS^rA8LA9udSqrsBmM$Xu7@_ zq8rEwT`#$%^Wo!3M8zy{dN`Sh|G($g~%7)m&6z8fk^P39N7yjEtGHYB&6wO zpY8m9N!5wIpa{o`3!xHmHF*9yqApbJO)k`_`DgJP3Va1F32_aHLNGD4HsyJ& zK&*1lh7rhtO#R)0kbsB{=^}M&mfWkZ=S=T`l*twT3c~dw?r*QNg_O$_^yrSBtGrU8 zVS6B|rN3Y6P;3jFW7Ls99yq47=uTiR$q4(5i^2Fr}bL z=Q{X=eI1{c3)c`=5I61p=U$&UN!lk$&&U0Cgx3{R6*1(P6p_r;l29U~C9~p2B8Pd* z$O~x;(Fgt(2Z~kTa;t zBo9$NRCvg2`*k!bJ~>r9oPVr_Q(|8FvNW-%Uh}kKf7d>eC7hOXT$NiA(!H0Yj6s$EzbR4V6Mbbv4)|4bP%>~mYb zUe)V&XH{pwb0d%2GoG`qGwHLebJX+Qa}XgWS^^>sq9&RK+Pi1YXqyC#?8WR}dLqmQ z1SsJ(gw=%JY#gs6*>x=E%%DF+=0931=qXs9^n5YcF;6nqu=W^wX4%Ci$vJE6GE2na z%7&kynz&fu$B;t*5wDS+$JEfIu^(@sG;Q8cqQ*#lLv2FiLUXv}ZSm6*f)X7~LJev) z+ZvnNdz)3WSbfLu8{bMot)s2KSy%3B@;dUCYRApaLHpB-SKu<>uynHu&l>|m5< z=yOC-zamU@W}6!>NPh6l1$lT*&wftzN1=Tk(#fEha!gyg_K@i?lFeRx*p?G=aKrq?nBIVBTE9T&#PqduCJe^~Uw? zNjm-&KC}wH`D0_YbHT!hvgB=i`PY3-bOu?Sg@zw`qQjwWad31P5VF$uy62YPr1jO8 zZ^z4vlEbRPRPfM>QPdPQi{SUbmUX`)Ba)wFExz*z$X}6knGI5g7mTTUuZrknj3=-Lzc3`(-R=4g| zv|ZD;zcI+2x_RifQ13Q*c@XEh=gHZy;q2@p=6ZQSadCEj;-Nb?RQIs)W-y>Uhz_Jr z633_Fp>;2^D{*dX(mH<0eV)2|x!-!tcCy_LcksPvF{jm+nBL=^yUM-RrTYMCBfe=6 zG{~kk<-y@7?rMC>?ikeTyXR$nb2FB%ccS?57@=6gfdRu=ba2dxFHEu0UW8;e0jzA9 zCVfNMIu~qc@m?m0bJ!EE zNPXA4X}%+-6O2ef^eivdbS@a6MX?iCcLV_NC?CJ!Boru)006i&b0sw=HEAgxLtAS` zeIr`~V@5Y?JJ{U-0G}HV?5(x2lRntZ+RDa}$Bm!tk2`o^?;k&AA_M<%i<2cknVPg5 zSlHIV7|hPd#>h-2@DvON^Enuq@FQ$ZH-0iRCnq}|CMH)`S4LM>Mq39{rq|ru z+)T_YOe`!6usawW-EExo-56{f$^U%F@8^gZI~qEe+c}xr+JGOQt8ZZI?8Hw-_Sn(i zUw@9%*v@aWj<@B~c=`>2&Z=X5li;bH~wVlV|5NeKREzPCEn@i0KD_2g3#= z1fx#>{cRPNKRSFghlX@6@PE7IkA^~+LV#t_|K9%JPtDjz2K!ab$FPw8FP;5$Gp3Lm z(*Hi9$D7P}fyiL%+H5K0|7pzs^a+o-@PCdP06!B60E*8t(BORdzg9sA0Qb@Le_Wrx z`|__L_XCW$NcMjGzlRQ6CgT4r(|?xfKi2dgr~6NA`cDY|Pp111Aovd;_z!OS|Av<@ z=zz8KZuH)VgSR82-gk!V@ON+Zq>!QO$9bcWLr<>e-D}O<)!IcTsje3hGU*40(u?&E z6K9)wAmWIFcXQpb-eDsUl552o#|@M1JI6~xdryMD(d-|YsYDb2mt8&a#PThuip|~(7NjO`^seDOcz8z?4}CgM=wyV@!a&rER}oI(gUD0f0T(>a@=S|3cn+naFBTx6T~ZygbhkXhJ@_-rO+huleE6X+E;M zC}7!hCK>8^JBgc!OeT$Gf@mzkxx2Nh{hi&K58PA#mA@2&;ozfj;l)^g^Nd_X|GtA8 zOOJJ5C@sIDwdZ-}fN9?4(4vWjRGPYs1yJyA5n(&Mv>{6${fT+kS=!kWFNS$f@>Mpu zpbq)*YLeIiilNH>p2P#+VUGE{c~MlD{U!gY8qb~YJuAZwB(t&D$0_?PU|uJ=Ml%38 z+Fl(V(f?n3;`<9MA;3%WGAGo(GXD29>LXojK^>2u%I6~Ja|ZKg?c|%2Llod&h44v> z^9Gw5qm83K@UVr_ekBq{LhBBOJ=;6R8$3p?c|ox*8jkG-CkYFO-a^hq&|ojQvJDRx!0f{ zqY7cd#mb-E`5~aApY)$inE3!>RhsnVvA>y)9vh0+!{udnjcU%2aE+Fr_vR6QFS#pn z6}`X7v0M%bN*PFy8BEGs9Kh#29N7zRyV`R>Omff zj2O*HmOS)o<5X6BbHUTQ;f$#@;+ztgZLI6~Ea2bEUWNkpdq#+h`+H;wPvJ2IbAQm* zZW1Lg&QFIz>kjvkx{Wj?9;bWejGsMA;tRH_ys{r=?9SN?A9W|h6C}DW*E^1V-9j$d z(mX3c;Cpwr@Nf+W6!*OE4p4vd4~mzPhfz{Rr{zSB-|H-iZRP@Qwb(@)$$>0*{|sFd z%IHL9Yi-;u;o-l#-e)Q(vBHJRR);1G^Yd$$otEQJ`#3q}9v)sKRY_Nm;{wI+dZT}@ z-(x>QnlO%S9%gKW{hOy3)BD@bAICd&Jls0ZqpgTVd=Lr4G4pYq)V0=VxU}Xz$x2vw z3hW_omd!JC-Vh6cmR;HZWbBsoO3>~n8odRjc`>8&LC6Iue4u%Q*HaeN>gRRHVD_hf zckt-3W~h%`yF6S9_?K1r$C3Wp6M6?#dzn;4EDIbsJQ(S~m}MOX26sl0J8yIyrm11N zQN4w7tu&mD2#2MI9ye0@Tt%P0J8nln2Kxsw{r1Youu_EvbJZI4_f)^-h3}!5byqc> zqxOr+e6njHJ#1bWdGeV&-`gai0Q@{}dB#!k39sj+e z&)_(Qf-Ldi-LMPI?8`u@Xeibf3SU=cNw4`c%_A<52DV)qK|s zPYueYuqrrnE9GQ)yeih%Dqvl{)1QML0H{!+H+uVzi3sV9V8i;Wk_=k@r$2uaq?6-W z+oVCC=|Q?tN^4SP#>Wt_9s=U9AcK#Fd56o8#dfB4WHR%yVNypFNFeM=Uj`YR`yJt5 z^uhmv6sA51SpJmy9jjkZ`8(RSP&Z#7O-@#030gd-&`1$FZEU;_A0Ihf^jdN#A+d!s zYRjY^k)LrX6!B4m);Ue$jF|KVH>n-Kzf)VD=m!1r-^CyM48jxisn{eO=^CB%m`KAE zO2fqn{5}51)l6`L#}uNPkI`rTZTWP<8PopZSLG~h=DcZ-vsn%W)!6ePV1|K$v^{Ma zHn+uqLn6T-vEA?I-M4t1SM9ihS})-!qL2T}oN6BD#3A`ppk9|jzeVq)o z_nmod((ccfQg=+TG0l{LbmTs6GOLt+bC>;w+pj_}M4@WFGX&cP)}HJ-pFx}g7x7L% z&hf`rx@lAAw_7i~?i|PkdmG68o@3Abjt`IC_KO5In;ApmM1KDklopv_M~ne8iEgaPwD<$>jp4F^m7v<@8MnBXw>+d+Q|Lzyh=aJ!Y2U-7))dLuecPUp353v5Bg#a(Ih`LuDdIR;=)@aN%wnK=N+APqwf zIq^uVfA8E&-~hT2nV`xiD(mi8m>u%kP9$yDLS}sfrC-oJuH!P#5%gX(D##j#)Q8Bo zU35F~e&NO9O83yEhhx%s<;>Iaw*YlgE4F{@bV7YFe=?uWX=DC9HI+}_^GfSkn-5KE zC$ypI^BcKQhppasu-H}dK0_VqcCg1jibf5oyIOXS3voZ_lizMme$ zGQ!PrA!S+`xuPVtBt6ByMhudx47X_(LEXo zTL(tv3+q23h9Dlj4~v{8^a_L6O~7-er#4mNb9Kd)eK9&8Q+#Js(WL<8bqfnVu&4J| z7Q=6odwixw6fB4CmZ0GM-pn#=cmap?{q6RfFS~~q3mo3!AV%aWp2q~5cW#nxsav_2 zpTM646tBE`ZCdR-(#Z?r)nDmO(jFaUy7bw79|l5<0I<+P2vh$Z12c>t&7)sI?{9oS zC%M!dzZfHMx4Sqw?`74xnLX;VSomiIKElA>htf>i5SFHv^5`#~)QwyYV+-9Mtp(UG zxj*KY;|T3Y+1<0^h7k_*yZ}0qCc4!8CqsS4$E&KhO$;R?)7lj(JCT42g)6gX|8|0} zkLsy|VG5Iuk>ciA{BzO&Na9t*kI6|>feQZ5p(c31ClU9E_((Oo>QxNZ zUC(>BG%mP6Bh`|;0=VEE+_DTwRopQ!(4A%y{~o=wdc>9t>}`dC>ZM}RD^AYCx|}fHGj66wGkCkPe<F9p`-O<3j>`_s4>CN%xfGfpW>39 zD9di~zr2S4-T%HwK;XsoH&!zq7Z2?nHRF5eX3$x=n6#B8g=! zddN}!wvL$~m?PX4Q?mXAsikOuwl~WW+eMsrw)~q+twwogH>n(ubP1shWfa&5kZkLH zZWfje8z1gNk-@bB_d+}>vpR7-7-1m%eFv+Pw$)YpyDf-ocI_Or#3SDshJH+*bz)$A z1SqUh9n-_#KyDH`qGlZ<2Ghp4ANXlXn#U`v%p%Crm09fOTRC2=J`v_o=-$8qhy8tGFT1se7c-pLlaKue*96 z3Q_>}aZ7tl=;v1OGcZ_XU4w3;i>bIj<>iY4rskWiqWK5Z#(sSyP1-!qg8sz0Qqq6M z)!YqF=9i}Pu$_p%(pMqJ60-5yW7__>%T(AIMLanNPHj9LUd6jUo=38gJZxI$J;xq8 zQ0LdK9GczPESCcNg;UP|Ty&ic23cBv=5gf`IM+QMR(lQ~{qwp*#B-gxa@HkZLGPc6_0AR7nr=Ayz zRe@fO#+fgDf}o#FsA#8p`hs(Ue9~_q(C(hYsTIGmvZ2T#m=R=4MDmB`OuidGh_3sEsbhII~{rB%6?MTyD&#?p$^SiB&hr z{Y;~;q3hGlv`nV)KWIm5ZxjNckU{feZPQ2>gT#Pm*_=(+*4sr7_lH^hWOorbu;^wF z9AM6p$+({9_qmlGnP!?b9_x0mK@TE-^rOc9DK76jcQKiStMdCZ_EBZe=JV-X_8pi} zWkpso@L2I4w-=rKN`*yVnu$EaFnkFBcPYa}sKnpF-QNUej{_oZpJ3Ys3+ad6wlfZc zDOHoS%>{}pVy3F9J9~Rl+S;iFS(W4mGOjKzIb$@oRjEsO5>R<}Q!k!4b@#$JTddCW zB&OP9b~ze^_4<}ZdW$ut+1I}oLGCuAQ!?aYIX-c>z1ryei|?HBs|REXCl@D^Z*Jz& zPpo`{Hm3AU%d^69h*fnN73l`1S83-3GwyE`b!}~@@*!1j_PRU!+9@9Rs_uo{nBxPo z)}7^&Uc3z*Q^olQEulPD&qD<_GKT|l&k(F)j7A?dY+{#O`Q&%IV;5JK4c0Z?5C=afsec!tlo}! zsINU(hpO%+MJCnS{{<&-VM$&+4dAd(ewiC<(^MDvNUzdrnl*ANJ+*%MoVFy7Hrn3$ z@K3;yE(sA_whj=RDudPT26ElM#&_hU{uyL!j3x9G<=}V9tKx>mc@AB@TqOTts-pwf zR$lAI{hDoOh_`ja^_;KzC%_0USupWc!HM;(zU`q$9s)e~>3;a7>}; zSwEAz{5&xm*A@fE)Jq)!;Tagm9D{hRt(T8gGvavnnG5`y#2<95|HWYJ2E2(J$+vk! z!FO$;m~v8IRk(Vc6fhGA7@=zu^3D|)udxGfLRUJ&-U7PU7aP3 zD!`eamDi#$-=%l$giFblXdn1^)dn(FqSffQTy-PF=qssF?O4dqGOW;A-~D>U_(j9p zgdC0{421W_qq1jMAFKUGcz>+j==^A^hlj_HuTN{mB4l5@q;wxJKsWX)blA|RLxjA$ zAG9vUsVTHJ?V&%QM+Mv796d;j5%UQWdz}3=?@BqoE82b2Cs6_;XHmG6VvQkGFszVP z)o!v~#N@s3G2z_mk(935*1h7-T3m0T(tu9c-Gn=>>)0k@!Z;Mphh}!9>MQ$mYH|g# zXM1}Rsi%vvF*VT^=aw+Am{1*ICFq?$jd9O-{&3RYvzy?kw=;D=^=+N+HzWi2bCAMt z3l-{5HZU^@JRZGTbjxY#;TB{|R^~3$;1y6j{{F|JftZNPl)SEhLz8Q-B1DsGem@xG zv@-h|0zFz%ML%AF+9XJ28Mb8aT13cr7QOIrN!^1VvN?XU**w2dpL0t4;K6Q3@=N>b zX{Fa>B{B45PpnG#c|-gsd;WAY71spIdYnsg8t_Lz?Y1hscshKv4+g|#9>R^= zSoDTarI6!WvEfg%;@I2rmv{zq(?Si`u9w-j_yQ>x^Tv%;6Dpo$e$VY<*B17Ks7?%H z=!r~nH(W2zd9Bl0pfRab;r$ix(M6yZ-sX!g+p(2y5>`%0-P!G6`!HzT)(~ed#@u~- zQeRh5zj$kc;Omv32q8d0ynzKZ6-=?Y92)&Dtqam>RiBlBx*aY#N|64W*_k;$iWn{5 zr7(b55=+BkAZb>U{$R%2YU@$kq3if221P6(qG^3nk;p?v-lp}GLx=ni2g|>r8rh># z@MnTz?r!M!EuIsklooOC3lNl&IdQdSmry;kU*M(BY;YhJoZZ)H+Hcw9`NqCAEKh*P z`1VVFD3a%~*LsgD-K4W%PUJqF`LP{g7@;scrkLCrEl>n$;KGuwSiqLXxVyPgJ^($0 zgg-Rn2v0#f6o`T*Z0x>+Cy~N|Z_E)2Cdt>5x6vuJx=_^mcdW0=a`wai;IkE61m2`cY(gR_oA7g zN{4WBJG3@qnt2_gVDq|2QPuIJP3&x;v?Xce2{QN-S4e4ltF!ctZ)+o@=86z9!_e%J zjJEqH$sSngPEM;8y;*s~E0bR@i6cbsLsax9zJCkjH#(|CA)qu)V2@jg4{8)EEJCjO z9=VA6dC}u#8V$prO0-mS_3fehyDD zOLx>F_3WcduV<~;R2B=bv10ZAiU7Zs}wF5%C<>#IRJ@ z8mjPSZQ!ycP5q6Q>00sC2e1gO5mYv;6bbf!Iuesvo4p(NR7^~nlo<`F?oC_(_MhIi z#_N1tu$%Ia3pTbl2M^3y2w;J68&z6KeX4Gr>MvG2>WL*_pQ#CxMwbwPY3J_He+VSSTJ zz5QkujvHT`ImUQbQSd-d2l+-Z^60Rw>!E-tCD!G-{xsWpdZC>t}$+2 z3gw7An}~ZNz_(SXPVG?irqThy7enc-c*5Fo`=>VfOp|R{;^({cC|n{P!gUy7h~)@h zS#BiZ#j%9G;OFN1m?DD$A?V-G5V8rFcvQaxbQcJal*x#(8-L_)NLHEJeSbrJOo_@D za1pPB#q-otkXQS}60U=+$8YGq3^|i~+Ai$Py_sHZITs!!V~k7W58 z&zHa&1Mwh-BGkTtb4IxWX_5`&A?ebW(fep1D5pNzCgXL#R#8R^rqI*!i5AdfEd=A< z&ewXa7GsvY+3MitAIxjr*e_v9Tqi7-OVjHjeX>dwmy79NLtf%w_d)Revc^ z9eSl7fGMxwuhyXKGdZ{8Rx4^+62S)18%X*cUGjcqs_y zRB#df4?*e*hLM67{yCb+RZskDo7Nf37rif~yGa_?z6B2e#a%`GUp0IOigVQ?)wVwk zYD_&O^$HFrI*Lr`8FUbwC5On4Dih5Q+R#TpbH;3~Y)m~@anT)6z%C!UITYo$cy8FZ zr}tZiR{TVt2yTxIc-MKK?O(IaWIzi|l%j@<4h)-KymN+0N*t;O%&6ez8DO8LVf|s* z5I=I%%j>=H;L^{(){b{x8#138h8OB=qtH&)gEgL9Dh8#E;Y~X{>WGar4Qa6T~p!-d4WArXfiMSvxC7N!Iy7Wf&wdEXdb>>O{CB_S^5<}BkDOT z_Y(KitSsv__eVGWkS{V`i>9Nqk^Efy<*O5kd7V-3;S=l6b<7#i-`I9cIhc1uebvN% zMnDAwdCb7-6;49#k8 zd8=pF;}7ZSkFzjs5RG9;3MF!q=W!VO7rZHc85D_0cpOCY*Ws>5Czz~+joeO z^Ly?V35brKslNzmDO{M-F*N$LVy-XY`My(~WIR}3G8oOBi}SeKpdE8LLqARBeP#$k zw3UwaLo`QPz+Q-CDo1I^Cg!G(8od2kklR<`arUOYi>gkEOGZ&7?zgWdrbLf{J)d?& zgGoQsD@S|T1SQ6OOSZPi&@RzcQyaQWv3;Po7Ick0gKiEwbACSDiV6k{6T?xQT8tmU zByL|4Zoy`^FlWtit-TLI(lF9v%?|T)4ZyWz_Y?ExlEZ@i1>cFevvFb!NP0KVaarm* z!Hjo!|M^g)M<23-shH052}5?C+1>t6B2`1g8*?iRm$9hcr?N7iSeqTz`>;x16T)Nm zy}T~e+EwnX6BYSz`_nm;RiLXjI^*MQZ~Nw_dq)0utsc?+pIMcMz1{lqxwz-eQ9$0! z+(|!-uf-Tjn(`lrq!kl1)MDzsf$mNwmXBws`e;9uTE*h9@l)9HPMd1jM}V2G8?Ob-+%!i*`} zo<-~L`k}P?eOz&BtyjLNRAC}(P<18k7B5?xGlAHv;^m5wO2xDbxnF{cr4Ev=gL1=O zY3oE6sn6Ie@hYxI|2X5Xl>m9SxiFZX&z;_rqgi>GTI0itOa|gG-$0JM@P$cD4Y_%P z{LVIAGn&=VIi#f8W)0uXvGUp3Alu6inv{NCEIY3K-UaZ{WUsMj7C|u)Cx{w zH8A?QW|%u~aay5kHJVTTLu8qvXemT?u%z$?&yKZnOJh|{#J*uH!cpSPsO@_qsg82E zq-;%T|8$&(u`AIEz$jgk9S>nRA1N$_tBj5FX&0l&NnCMzKViT&Y9dNhK=Ks(t=ZWYvm*^fj450s(EbX@JL70 z_2Kw~TEghb>C3A-jM4Ab@nbG7R%MbqZuc-1NS_i@=u=?G``?=+NdG4dE{=N#(2IZ(j@do#lz_dcq-@z%g-aGYz4d^O=UoHD2s!L2@=NU%Am5)CWU9s2y$+P=WfZcs8Yt5Ch_=p|L;!j@us z%|d$+z<^YWOx!xs2qlcPT*VzoA{UQ9IJ=anYREg;r5rzO+*KMs8hDL$2o<@hjcQ^a zP%|7?&|an6F4d}E){q4DBqa+%NQ2fyaNLFiRddr9E6X@mE3z)%-&%7t{;8EStHY{w zcju93Uz1yv4T)@zS6O|V$h_}M>2{CPISsSU2DD}K(={3V0V5W4EblI)oR8bp9y}y> z9>(*4cHJsL*^&EQw!BTRHIy5EpOF9nB`N*>tSjp6U~wJPt&Y6zCW_U$KZ(b%Jc$4X z52b8f;HjPP1Oc#zpE@|sP7}*GaaU4$bWEuA+IbWJgT-v@Jc3B$9%j%_90H#-^Ia{> zb*aZ)HjNGJ7qz#|Q>C1@aE6e%3^lfnJ54t=w6pyk!T$W42qS7Rf`^L=C1d z(HC)!vz)&oNu~t~tv2pjnC?|--z@3O@=1n;&?K21ulDmY$?Dc`CV8BcF#1f%!AE=4 z&u|XQtaNuZ=0&D1HW^r+PbR~H7tf4~)j10aIH*KTu<9RikwFF~E_UK0pFUFl`}@NO zf&Kc;wC!kN^p_WQ1F$V{@4*6dTeP7cabbv%Z0~hIPS)oybT`F}goB^Jq#H|Z_h#X; zS848=irnRNoNm^-SR6ayIg|vqAt$}pI^6A*vfgGn@5CPb5CXq-?&NTj6TXPNE?hKt zU#09UCMp6SJQk6^SyH<4%P|DOdvH`cKG`!33kADw*?D9bhnP!Legvkx_WPKhsM?G~O3abY22+;~Qm0hCTb2oUnh7&c-QHuwr zfs$@iXC8&MOwCY2)yY4K#$IDO=8%ZVM7hQnM8H0Bm^5?ax)5}kI2p6j?Fy=6Xg>XV zIHu@v?QuG?StD5GL$cdp^jxqX9?qzEF&yQcxw;rC1s{pp(Gk!dJi-3B8|O zd=I#ms@k2}S;rRYvej(*Oel%^TcQS+t-un$N(wOp=`RlW(w3VYOrduyq^RJ@+h>^E zYpYNU6tMk4P!?}tD|Bd9=O-5NFsh4Jyf|VQ_#)Djx*8$cPQDvPS1`&&13TLU5+J>j zTEdp=1mmuA6@ZZ~Oc3&Uj>FfYNa8bL#xOQ#6`8T9!WZO>oh|t|3()uX$fhl8z!0#& z6<_+@XAEHsI111Ec6D1{Z;@~}y~H_lcG4F-nYk-jWMLrgllE;yR#s_sd=;fpnr*LI zzzA10CO#kGdVCdjwHxLKvGC=KnzmY^dilZoV1FyGFW05ArMa%)%BXOntmPQ!i&P10 z`8aV}PAjt)i@1cRDfjLCmwDXD#rURqYmq0lLHl3Ulo*qhN~>#=hF*dZy5Q&cg#@-T zb#sgEp>k?|SFhiowv~}TDrPsA4}vXA+}p5>PhziYIIox48O3^V1#uc|o*W;Y^DdDd z6>TKa6s&as!ez&)no?RP_pfx9#Ao{pLdy`#MplA6J%oTg>~VmKb2^3Y8c69aHQYb- zn-Ez2hPQ;u(f>O#3iKS}E4^FVcz~?Az9R}K?^zBgZEOey#MHL!jF4`**tuRYJMkW_ zPouvP*=|ZzbjfT~sVj6mpUXY@fWgy#n&4yOWU@K8_d-{rp=v1d!b-RR-!%l($yqwm zkZ-g31M?+fpH=~1{Z!tnKaD!F(-&zO12cGptg5jtFBX zNtYGnpgID5m?xwGfjxE2w5zm2TzIaXgCg%&niwzGd1@qeTuOM$41R6zvs}hDp3I|PwzXs}(gG=- z614oN>TUUCmCU_N-1j#slD&FMwqec1;J4o^}x zHXJXvc|sF45v@2Lf{a=7odBLSk_J{Fq&->XyyPd!(ty_OZ8eYD5DpTF(Y<&IB&YsG zKX>gq)O}$qeY~1{IiIrNJ$tz;Ayakr+IM4P8=4wbx}W$A1y&a;drw}4b!@o(oup14 z*khViY`80o3wKhOGf9sB*EuOTWGRP74CMv)8U&9Q1#Ln|9PrLm%>CGxNnJpV zHB*MCLUZnI@NI-Z(NbVr5;yL_^-31xX{EAqJr6RHmt#8&Yf)?wZlUStiukHsy^;r) z>g<&s=ExqiP=_H6$Da}Y#F5{Djt%RENaDTvLX3zWN+bWea5AvIq{aJ-2TW^=U$aLF zj4}w?r?=|k5ka!iG@c@n1f6qJ>FXM2@eCy?;@Mkye|}d;a3JAKt4}+_X!wThsEyi} z%X>SVGiQJ!iP&Un@Z_gUoGpCx+f3yHm~5Cd+OXP>(GjJhPP(Ut2jdr$t&RfD-_f_n z=4-T6Vp00iVVU`^_JLU&x=FZueb_jda(12?S)5KS38Nbo#beC-DX^e3u$kxI=%XN8 zWjN?xV0bG(_x=DTcej?pM_W_)Sp@(T+Npj=9wAK_%P=omH)6q56Yv+t;Acy9nK9oE zWb#@*p%avhm;y>-f!$H*iT!k7VuxG&t0;P&sWxo^Y{?a`Xd+65lneBkGID-YlI70H z9t%Sx52ct@KIn=|@@;G+H0zH;VbH1O(bEY{ORr~o z&ykD4e{5VMY~vg?6`*zEKiZKfgud+NN4UH>&SPvq`mWG^T{`Ww z-i*@lJDHMHT6*lHJS==M8GI>7sA8AxUydheHF?J=4!G7*n<^imqx5TXKq$$1`Arsj{_>PZzl1Szl5JbK_Ztse=;k2^(S(EdF@slI@A>iAo6h5a^klu{Y3pM( zu%J_OfYqs|TRZyM6iWo;ei(+szcX`O;(OhvV<|XAW7}FIioH4QoqlRDm^>bFL zpsHmqwF~v``|1s~ph#T)e#_n3n?gzDqVg2^lXm9wrI59~77IeU%iaDuu~}N`3SJ7Y zv)qM+g|WOR8LR!_7R4UPG+`7sZrOaYlw+E~;wxjy1w`(=ug|TYbk5G-_C+0MG~T|N z%M#>|fB7o0HYf&Q7|IwdZyY84)V%F|aOso`{a(2Im)OlQ5w4R{u6k|bH{SO_w-sAo zB(+BoZ}D`G1+KV3h_>b(bSZsbq-q!3Ws}M`&Z>UiR0c#`Cul~>$@}VOS?(yXwvrvI z&@46^$izP-#jUF<*|O#F=)0Z46yhA7;|JfTMoi)Z%dyaBWZoZhAdlCvBxwSpDHWUP|tKp zEdR9r?Kn$(XYJ5MK5dX%2BJ@~*MZEsh4Sq?kViNLlSfEIaHun?pu0^ZpjY| zGB@dou$LSR>+hmv>!5;H4v{fyE2D7)lSG2SHNpsZaC}27v)H1eG;kC*amu7D*V9kj z)MRO8O^?)#U)OY(rw+I7swh+VZM6b^eBvt&Vj3b5L`90!u+PfCKXPc!m-EF{n07Gb zU0-l`Z@9y+wo);$!-}dX@4YVBko2l(twlDqtrVz79pt5{2H z$kMjbRl{=dl+YrUhz^Ls->hcpqop1OV$HkE?u(ob>xn7{%Cu&^(}QJyYMUlW9>Zk( zD1xwc=P`PpCbx@;ZC%%^1i+0)YroTxU%VJ+qVhJ(x%gAD`&1 zW9?%BgHz9^x968@dQlgh5@NL%s&TZG_uDUUTdc;z30zXdbk@rtL}xBn@sce5cNg~S z_h-y%MAtda=8705VMOob#vR(*a;Li*D!VKS4?Q9+?6cqDF~x{P-3Q+XBV(eFzFyvv z!!t7w6*;!^2yH48_5Qf^tzp=7>ho#lVR0Fsr(=L>nt*%cjysrJS zsZ7tia(!a~0^YD^-0hozN(4iC#Dsf&28#2{dA)V=r7--*Rl6K|?QWA6J7X+Lx3Rp* zI@kdAz8YICM6@Z8E$9459wxyJIZ%#DbN0&2Pcl^2Cx(wE7+0f$E%p(VZlnYi6sAw_ z1BKRkk6ctv67}hC69Va^Gt}!&AqgrG|0w$Wn<03DI`wTA+pMQLIrJOvcud~e#)kDh2$?`J@6^LVPt;MnCfaB6h4YiIst(e9_4~nywqdw&KUh3V_r-@) zvl6gm4iqvxX$X_j1N{{S?q+dG9uOO=ADT$sNoZg5#vCiHH)Qq|)>P;WT`MZ21^as8 zHfQt2mlSHLWa3r;4D!4?4D&@k_z_({c_MK8gMrRJgO=$uT%IjFc---_Or6W_&GX)r zp5ja!CB_Yt{ltz7jn~xcblpyDbpm)|MzUC-6bRkSbqe-o5yqtB!jdw7fZJi16`mzVwolz`n_1sM06Y~c{!RVG-Ti^5vEpO%8=;m=mjf2r1>XYWI`-?<<~k#W0_9h5mYBux{H6TWNPZl)wOOI)2b2au#TJeF!GGinV>J} zbWXKyGz&sI|LgnU=^OhFS)12qRTewhzyB8i?FAPB`Q^F~y?@7OM*HHRFFOMznvb{V zHhbP)Rjts{D-2Z9NW>6UA%{SXgjcIy1{oCA(wsSK%QBLp)Ycp1fy2C z-m&Di{XDhjd|`Vl^kTj}g_zi0;*AIhx`WQBVZ=3)ntJOo2{A=$eQ{3QEFGt(d1d|D z#x=t#&g8T?I|XN({wI9nK^rfwZ+MpKFtQ~d*6++O`rFL>`LyO)P{j-getdH4Xx>?@ zd2ZjW&WJkuC9QhC1sPoKV0a2U|A_!QT)W!VZ3qYhJ-`kIT2HDQS|MhuB8Ox9n=B8u zzH7M|nRqVP;&92n-M^@-GXzCTK1f?=9lt+6RimC!hcffRN0<8qbmc#|U<=l`2M(^f zoxkw6*t%e9mAISqp<&^v;gtgR^btIj_%pbqIw%*QH4ZkDI(RQ787QZ*+ zXL|~Uou6t#`+9eFT)xxM*l+XSw|*c(PqovmU^mvE&zI;HvQ1nnISCtX$I zwi_4xZ)jILL>!4v&zE%Tue!I70xsY07B_EzkuG;R4#XZ<(9UgU4T`OF4c*0@Il^{> zmez6$E#Zvz>$8}UCmfT1pmaphG+o7d?F<~2=ABl@@2AL2wVaa)!p_dPo!Yi0`ZO)M zo`-^}mm|0P_jNP`YlUI(9J_SI2v5-Fs+R32Pj}*%H)cHs1t=c%zy943jx4OAyfO(u z1v68@<1uO_m<%MbQ+YRzb}^Lk^x>(wEZmY~I5YTtITie}o&Rxf%#~QR_eZeTsJ*&L zurH%!-H&T)q#7Ts?#%c0UKlElz6T?wp(5*TWUhrpPMYvv+MWt`J37xkcQJnps~iS~ zwS>Aw!&R>ek_jnwzx^(xODVUtWMk_-4hhEa4wN1a+gaZ)cUXHTVqx~32sg&M%KLS8 z>yI{2yQ=r)r?L@3Zxx{Ui(LJR#DYrA&+OPa4U~RP&b#zqxuQf@3+DE`4&wnMhb6VN zDkdgxNN;{6w9xx>T&Sg~#^EE}D^QoGR5Y`WVP8JSwRXr}opvu}Bs@{O=I*8Sw3bb? zGxg9g6)UdUIn$ane6?OAGFsO-M?agqA9IAGPyM>gvil`<9H~1xp2r!hXCpq?4}N&~ zuHhlXbmSgiM_tUPxOV2u>AKt#=lXR+!nXA@WO`{R*CB3WZq}wXA7a-3BkL{0qKw+M zQBnqxR6>cNq(izphLBQ9Qc6_1yK4v;8tD!}x*K6=1?leYj)4L8?fbpY^X+}?zZ~)d zuUXeR*I83$kPJMyMCxZmGK4L&68D`k%8CMSOejFqbneW0IV@{^Pb;Kl{S+`xC%KNH z6I=~#@ZZEwWMuwBqW!lMBXtjBSp4qbB6Yg;({HW>nZG{KVVE(Fpk_=0yO7B*MJ-~J zXI{xQ64g7gi__5tObBf}76A|UhGYV!gK0dP&Utf>uZcLVznG2FcQPDr@E>|Q=r9}) z*;vd8vuj6!v`vG&{$xO0h16u^2eFTZ6M2^I6~{qU-Gm?_LE*OJgj_B**Nf|~X{?c8 zz{!*!N_mbRF|(r{w9|(c90I;19p#FxA8M~3l+jKOroyJYIt2brZw=EZqe+2|&(~T4 zZ*KcZVv_i0zW~$P<=cHlLovN94s6;8JkyNhlpKM-?^yO) zJ3eS#hW{VQH58mUa#JXbLeDZJd$dyY%bGi#|wZlq$QgfdjJr z%TP+`wPn2 z%DtX5*t=j7~Kw zP(%MOb-fM&jv!K(hX|uLy))D6D5d?n?YLmJ#uA3mC6O$DyiKmH(MX2l3thYpG!JX< z!EdTMv>(abvA=0y$_4v^W5IS6oHE+$zjbW9NK4AIKniCGQ$cE z%Wq|F)Pv=T?@b)w(bs~EGKM6hooJAhuw%JrKi=Y$^7_vSKc&l}B-(axe`~=prAs13 zQD!ns!yjG&d=vvL?%LQtxWO<9Y=SnSF}nC8Jx=86DOxNedRa>LT%3_4E*p^7uNwQ_ z74FTt;s|litcB4h{g$ziD}ERGQ^K8U@ObK`z53ZIh2L4-NGw=EqU9v-%Utbly}>=p za@H^X$&TBIPg-{?IkiVLyjml+d&Mu7DTRonRr!)W|Z0PIKt;LehQ#PxAB7A zvyhI8J_(0-!||8;U|op@ePFM~{oDa`yEav|FTJ1GB?4Z+HSauE z3O!qg7nf7UJVhpf^yq&An3663n2!cx3-R|!okuOlnZsh&3$*}LiN-+(sPK$?xo~jw zo0u9yLM#V7D01P!>QV4wPTbBTM~18l-5~Q&&mZbM%t}bTdX9JO*8O5*0?)1C*wtQQ ze$!zRa1GFsHL4JjH+%0i`-!@Ct2w7&;d6Dle&sf)7^g`ujzec)`~2B|CVDWIvMrLN z2vyF9ZE6us0rY(ET3c5`ryly2FIc`V;An~8J-zV-C2dttat$&(Wv>*#3YeOFK6YiB z-fkOEQXJ|WDW*OL8V&YbNBTES258XfUL{CWUbwSbynO-RImrj15z_!Q%&O1tG4La? znmPohWi1eAqNW$$KR|I4ZR%Tre8T_-t@4JJ)o!&9OjPWC1L>zw`ZkL}#4}J)9UCv! z@wtonO+iC?Vj@!krAJ)5Ak?PW8{8xR^ZN znT3V<2M-S8@-rm+zhNnA{}VjPY+O+5WtpAIvFCKl*p$AVm~BlJ%2`^44-H0)tUtMeEOA2l~j@>3j6VxFOP;*v%~uT_l+eaoFtI z>Q(yVCelx+?YxOV>-trTGFBo!p@enhTnMgKG6}kMku<{#4UMRGxFR!chnljUx+^i` zZ4tL8^*SWd-W*Ds;UjK$2R85D!YSb&e^J|EZ$4xQ>3-sg0IfM&(`_~ZqKxnCs3oi; zRo~yyIZW&!iP5_}j7|zk=$<%m)u@kD*K?xUp-?x6(#bZ{8(hN0Zq=WEdm$W@=g8)} zA@et}qV|r12x?OWT^}R~ZJqe06J^Y8pt2h{?vpYUYj-Crd_8{rcDzU>vWuWtlX>rl z%1ax_uI>uwQ!l#f3A)(yw5AU{?g~@%0e2}Ct0%xrXgJoUeFl=L z<%fu=<%3wGs>$2uHgiPlSq^=E1x`n8<-=|hA$gp=`@*N8k>kYpi(2?@XACU`__jWh z_Y36!!AoKcsJG5N#5(ytS1%C=31wN|0|z~&ecRT6s+v|cXjK0t(!*w?~rxxl(!MMg$L$Q?K@_KKGur=S$ zW~?5-R-N3XKvYXhgI#_85`@^f3eU3cXf(QFK?vP%gW9DiiRWT(@khI8Rrp+sSs3WC zd`?H_PdQ6P@gdr}(Ejlk0VrQhAm`)uww9~3w1VahefF7~Av>}jRiLk=@9PNGhVOTi z+-X+_u?nWc3c@7U92?#nzY>8nUX$mB+>*o)C>pRo`F*uH`se*~heyOvxWTB$<%&^O zfM8$#5Pyi_oKk5-bNFf=EY!$!!8=-M_)m??wS#!X(B-JkYkzmN>gk{6UQMl-Q@@)) z)FFKl3y(U4;5)1NQX)iE3avQDbiIzO?@y4*TofaaZ|m;()E<6QHSrp7oGdlYQte(e z-Lco%wph^m*PcYw$KH%K-H$T=5&8_TonD9WEu?B9LtC28Nb;hP75qv_TV`&$BGfM6MxfKtahy-ITZ*&oI z@ujadbG>DzAQzfhnaI-6xS~|P4u&1^)4;zMY;Qa`3w2DuYE&;M_*>#;OMJ6Ns2nc$ z;=SHi_)cwt2&VV{lIW61cwU5;Rn~?QYM4GTir4KBE$m<~6D*N;RO{)L-+!JOCO;6W zeYI5;f$MvAQolK1Nm4LbyQx(*SLYD!UVHxT$)68w$mb-YcL3ismE zxIE`tt?yJ?`Bz^BtXkS!K*DGWA&zhX0NG>$$Y#4QE4m&C3Og*ce_7|#dFf!x>)+`P zq|uI$`WqsUgA^lS%eGTqGhMT`$18AQ^}_=*{7X-*2F7z*x-NP3%FR zf0?Jva)@+l977cNI!PP?R-=z*|5Kt^s;D-_MJG z4q?$X@5ghsnKox^;}9u06m+YVj(+wdeQvm-+&%`<6Bkc8>&vnhcI+Zd8VqjTnB6}H z&&TxO2_*SUIteZv?j}3NQJx*q&s(nMS?9d4^a9v(pNJb!Y|pUYWL4(JRhc&TeEW$$ zHmyI&8Mo!H>bFw10~M?$phQ0!W%EN=BBSmcEraGQJZ40Vr=jOk>BW1p^dUEMyI|>< z2+<t6c%I_bK>#9x&x0~R?C4V33p>R*;Lld^_8>)G0MbN;3O_7aGIO&3- z;S^J~Iv@~_b4r;mo2jaGv|g-_@=t4u!SZftVwhKT;3Msh%c|`9DA@DA-w@r7b*N&$ zGs^q88on=$g8hsYc^MX#6@@+$3B*XtCf7QhfNX`lQnv%`{}3ue*Rbzdc&Rl zP4pCzCw9XPs$8ettN`hcymndmO6!~dPj;+tzdcLt5T!j(hJw$R_t8h?C$RoIAOB8s z-~@8d>JZ!&(FYthEQ{aoKqQpXtg8ot|11MivhjNiXg&McOKK8H3O<7Ew=?+dny9jv zS26p>BkBZY-<9)ic|?-ZPhxcz za|0XRMkL4)c-{Jh6lyS&c-Hoh>f2atI=%5e3T@q`aTh$v&db3c5KcrP%9EYDg5T_w zcF)`HHbrYf5nk$(P=q4=80ulH8LI5M;IK50_DE{QhvxeBE@`LHbHFfp;))?h;&`@0 zHN|%>Xq(`rEzNM5?7~e7IVrXw6l05VMw~cTSp6vSiasNNEtECawhd3e`K2Q;zh0?FLg9DOqk&KK`g#$z-&eOr8{RdCCv(!Zm zJER@O>+R`;#$R7OfwN_l4sBZS^RrwZ#tF{LbcI6tr6Y-`x&wo5Pn+l-b~CrmrCyl4 zh^1vYU)GSyplM6<53cyR z7`pym+Qm#8{z}7I!(6m@b$?NIXIp;w3UH6z>~ktji+S2b9WQ@>uZ4Vrf)V46tG8|i z$OF^qiDT#jlN2IuqFe=ku1d4Q(r)&2I#O-h5nC52Bi^4HRdSaIPV;IQU5-vUXeH!j zV+Q|&VQQZPsWVqSm94QWj7;ydfs36ySRpR|%*U>6y?^lY2#Zeg%=nBoTV~0A0R8O! z^}gnbJyc|nu624g0F0lwK95-Ky#ARRcT^YD;kY(W_a4#KEoS#i+V&cHWA6e{DF%{O z%_5XOf`nfLl1;8+_v3xG`QL~P+?5=b=bB9N(~7505%VjFo=#YYh9oqy_AwXTJNOZr z^ASXzoHd6v$~9ND*VX)pKUdu`mMsdt@`3#;=_B_(Qw2uM$MQKY zw?B}={kJ4Oh%7qE^ljy<;mp35)GvEa+U@*K94Os=EhrIHe}t{_hc^EOo!pVdd=5Z6 zKxp4Xc)Q$gJ~_va9{fjJF_&SSLh}s>O(@Dcg|{bd*FPrwCK=y(Zj@Uq{Cn3|jK5E< zn#CZmN3PyMfIaJ~BYu1wpEZHH)!lZHwfkU2pOo+~)C^?S#^#|zppm;=QNr@#AOvAPjk)w07fM@5G2n2cO1_+O?#L2oX zPJgxCvq(NR`bjpfZ0;B2p4a)|6VxTAF)zCIQ$|mSNaT$|DY+J&xX591Y?5sWmn) z@TP9dYECB80R(S2e~IxVU{qI-l`s^iFdP*#8n2eqPMsfhgaS@puo!@)vJ7~ouzlMpWQGoft8u5(6U$6-K%<{ z6n2DoMq1$Wg9}YTo-5rh63i+17*TujLr!CHt5eqh8fq@70kvU8Lkr0+p^X+kK8Ng~P?s4Huqgg;mRIB#>1 z>}Isy*1d{9I&rz!u7(E9M`>`nfev=1+t*uK`XnDtIFSe&l!ISVZhcLcM}@C%eo;ub z2NQYRmX_6;t7OWMZ8J_M;tsoEL^$QX9nsk0w5us-v9|Zf8f14SOZDjcwb|_TkQuqX z!lS(QPDG!XC%623K`WfT>MvneRG%9q>7HY|=V;8_5^H_IyF#~}F+mz@#7{i-QaS-{ z)o*3>RY&ctkEcbMs#zk(^Sj4Oo^PEOCT&DS6>Q1TcZW<201ccJ14dt0wSSz{jap0rMo=5gmJB=8Ip;S+vFQQJ@5Hwo+GTZgXsrcCg+y1S` zbAK_@MktXP@=~r$>K~JHY>DIMWn1fNMK<$MPB|QW3lw;iU8~v6Im4dd6nz33UIM4v zGT^WbRn;%!KJw2le%1^t;gE8?1ta<)q01zvW;Z{J|C8S7OaO{6xHps1vDNs|0DFT+ z^JbF%*ek1_qwfm)NOUdz_-WBSn_n=mpx00$2=VMi%4)h>l)DsMzs`TL;{?*$u5ovn z`};Iw;_fX5S0>xAjA_+PR#_a8y7fHn#e1Cx8uOhcQt8mFO`Hav1xBC!qwR?z2YOHE zXZ;L|c`zvQcDa`9glal#43I6YFC>>w< zo8Hi^(tK~uB<<=*QhG-*;!6^V%YdEX2-CWsa2NqQ+(Pn@LXrEgg}$a@ujs?Y`&||A zgLm`I`ywDD=F&{8NF@3-R>o$bthdI0>W2`?^Rv=b$LkaoCueAKrxOn=DO_YD_#9b9 zMzUX8txG3XN?rdeLKyvvA!VRA4l10r`S;LT9}Y3%^n@oeFS-YoH5AZ##UhL!ii5`f z78qv@=+MoL8w!3rThJ*={@L=U@vqO%;}qw>VPr9##D&+HU4yTN=VD>Yx1pOdaS&QR zM)_Yr>!9$cTA6xfw0&-rHOAmIInu4--crh#=}zmjBLjYA8}F@-oXnz%Lhhty3SWzU zv#AJGi}`N=;=e&}fhKJ0`q@toC#z8}dGu^)NhEzQFGoTp3Gh5cDmQcR7#l=qA0mAq zAZNLoz}69mU0ZOIU+x0l$%*-Paj<=Rm_*A29t|}-a4_J|S=M_7`=}Tj=Sf>z)gR5Q zJTSe#`_PZ}p62~=MzAfECUP~_TiZT!699s>{Jh0vCpTUPq}HAaC8K;Pmd738{%B59mh-p9&Gt$j01Q-LpM4H>4J+K8)c|y{GjLPM)I;XGsDP z+*SI7aLymOi|pR3UEk6%HZ+v&v95(-q%64~DM!ATthz~As#Wm&)qe9boF=l?B>^7u zFhjyzkjX&_`q7X5t$3B)bjNJn`2z{lum|Ht+z6V&KmslaEz5H2ptTnX#$W>`#$zOm zWslTG)SZp-yy^9DyyS|L`ohw^rVqp;LC@;sh9y2=>6UevHQd+nj)2LaoD~*#t~_^L z*+3;y@<}S^>w3Cwk&-g@C_GfJ2vjA#OP1HndInGKkU@#H;j$Eayh+_3SH517lZGS* z&orx(b<*3jkr}Hsss(MC=>RPBymMJz}(ep{!Dh@#uGCEdCqpkrZ_V zRJ$ix^D5x3FpBB+dBov4)4_sUh$XS)vo1UvkE(NsttU36c!$}kX*f92>4ic?CMkP0 zO0}Ta`R`l;>Dw3gCa2v2RIV{YlT>FAYn#ngo5dH>sM#lRtcjRL>YJB;Z|u z4f3cl!Z}OeOv8U6VxAoBYuW{uWO?)DjiQj(C7&rPK3HRIiIPrS8!V%Rd!2|4qi7(; z+akxD@4?s5E6{4*`$ve8Ww-l2D-cF6eH-7oT2^`XYj5+OX_UqntGPBYbvfKql`HzC zvTPisMq$gJmyU86P`1HOAuE8nI+?-$r-{&q4KPJIt-)%sPv9|E$v^kAKC7BcH^XGWkRaBJnjg{@aXcBfL7-Hn+ zd%O4?$CT{Rty9AKJkGhRR|K|M8D9;!N*#>qfv~K>WmN*nk`em;!sQ=kn?^t_TY$f` zOegd^ai04!SO2w9r{-ogXzkh?KNj4*2@(76sQ)!8G1o1;b>${F2?+DMcJrG)D+I!b zmrYs8@8|q_DqwWPCG1g-FBHTZXo0O+c~94;c{AoRqJcZfdr5)iG=w-TZQS%}h+oW= zYHC%{6kJxufCj^8{n_xiWTpE}v8f$5IHubCV(FnuHzy>z;;r72;zs&oHbu?ii`C05 z>cli~3#zfK%IRfKs3@#7*w!R01Y0D|yV29(cZ(54FV(!(7c-VW67Y!ckqu^JjIu8v zSvzsKnemn+fQCl9y=7V?C+Ms6$GbwFxSLH|vn*F_do$WmQ_k0Qd5MM1DREH)Vtgc% zr1x<^gQeuT#o*OuEyoB0EvtFKIKJqqd-M6ob09$|Um_wdJ(HS3#i3-Cyo^h^FH;-z3-FoFZ{Q6@tNSKzL9ED*LaZ>IfK})0pp< zJ{NsX$Ld5le#LeeF(Tlm>Oz1V)qK~Wm!9;L&gUcu{1%7`*m`b@EDL$Vw_N|v>u8~O ztN5X3$f<7Fh%?Zppp%>a6H9jAo8xq3<;$f>%~ z6p`0ba~p}Q9IewD*j@SINdj6ZJvSg25!w~jD1^+fto?HJb(a4`ZduwNVX8(okhj}+ z9$B}eb}e?Ra62K;k@c{;k$3LFOd^3XkqdEy9mO8K9oLzB6iS{MeUDA&!O9kCK8rCj zM@UWM-2VyZz4(Th@!Z(p2sfv=i6z&cm}MCDz_0XTQcAsqs~=X3YtS2a_gh4RvRU{0>WEQdA2B#f` z>)1#xT~f7PJS@eN@S;DQsDCcfGI!5-zq&THh`5u%+q5bCZBiy$!yE{7g#kS6gjC@;_+RZnL!dIMqKX(7kd{R z1Dw3IT)*OpCoE!>>x1kKjW?H8*?+5iBp8H~_|xMu6E zvtU{y5{VWcZa{fhW5!?~UZ0J%M+LCox?+`25@r zf#X^Pu$w${p>gPQDqhY~kGIYp`CL_|+Q&X3^=(beKA>rTmM5yNWU70&+TuI81ISZ; z+YSzsBFSeU{b%l8>vO4CJm4vx@7Y+O6I^fnME5I;bW1*qSxovP979#~6z)7Mv>nyBD9c}o;y5`i z{=5t5q_oXi{utIEI&rD}&QYny_#$k5&cacBez%;JR#JNDZ&3Q0IDuCangaBRW70kq zA$9cG-v{jn!Q33cjo2LGnmvOyy9&JLZ-dcj$TgRWf=1iqw3uFBwU)*rPe%XdLhpBJ zz6)w>&ToSWUpoH0W3;O>r|GACTTipMK{D#skt+chNuSa;zjG_+Ve)l9$o1crA(Zm#WK_a&Xk6!SGlV71zL zm0%wDgF!geWQ|QEGT{N}m{K?v*{kIJxGEP57F%4rg?d@r6|_QE)NL{psI;BKYkJST z#tIHATetWD(Twt)f~jqJUNtvv+MXhlh;e8!E1QE}m$t>&%HqD}Y$~0rW7=hA4QE(( z&zrW3xYE6T7=pYk(OC-4k&7cyQJN9P_Q!FY@nQ4e%=d&2Jrvm zb5rHPNq1(3{Z9E4wLjZ{xu#Rn7Ceh>t#2~x?QV{U6-7o%n^?gm#5a2QM^R>~>w-g{ z^VtC7v=6&L)eSaKv3;6A)yAlk!InGf$S9BbpPQHZ&DFnTekK>4_Hb9!t+B}7*C63)ONt7QzVzl$aCJ=XSixs>-3(5CK@j^?9H$Ep0Z6WL6qP~Z z7--nkLb=lSy1^hoKXt^S->I~bdvU|jB)JxH#%`<2JL)PCJy$utaE)5`t@mrP{t&92 zW$U3crO2!gVyhp`r;GHsS}7Qyd82oVr{B9+7d32q7TP)R!oIW&kqN`Ha26^an`CeZ zW<9{`KicmuU}4w9U8==sCALmV;rkxA5sum0y3cWNsYD0)WXQ8fX{X&%FtOaq&9e>; z&%;m4i-yjtKd*z0NnE7OHCl9eu&F4CIz631o(&^6CpGg*TydiG1}Q4V@~_}qa`|DH zHDA+8CE57kU41L4yCwU!2FCXmr@x=$?b205=D_FnjL5B{W~YemHKqvu6={i|uYU=5 z18iqEzhPRhjU1%16aO*&fCzWjj6Swdr(-8n18aKdx+o0;gz)B%+h${#JOZUzecUFe z>y=g`D%Ya-3RLnPA4ldM{yOj6j=UT)nJ)ye6__kc**?9Il zf$Gu8Ht7Gel~K?D=D*7dB>{g#DbJ$;bBJ~Fk;uIzC@BbIzuv# z=a!T*U5NWnE3#(de!N%puq!jyX&z+PoeALJ^v#d-2XbuJrKuVna{&WEbLmhN2Vw91 z2S*8xcSf==XMlb!s6u!#LBVUP3l>UC|Hk%LvUa1q?Y!BCabma3XiAyy%k@_)vH5@3 zv2amK=zNt)Xj-r$3=Ea~rbG~KQzOcy1wzOpqU%re;I4+%B1#trLtS#V^E6y=r)nRc z-Eu4{1*?iG;2UwbQcbMFMiAep;)MZb#cDxtMs!LmeL|He=5`v%o+(QYWviql5$8>S z;P6w`Mz>el-0D8VzC+h2Klr*QFhwsZEVGs}ewu!W?`TxYWbd-MQ}fXJgA(5skCyuRdRXRy1BiXGgcQccGu1&KHvfO`ikT=H1<-n> zyVJ4{k4Q`Z?`od&PghGWOD&rVP9yhA#XQ?-PHKbw_z!N!Mi5{4<(4akf`NSxM|%B6 zAH$FDo;tD{SmId+u;9%yPk6hV+F82OaBKiN)?j}uomm#Sith$7tFu z_~^^GcPOh;VIn?DS8|Ycc}`tMQ3`=k3{U`96VJ*aLymv4(7Ooqzki_klYcPg=|4FV zVxo~tTAj?Wh&4yZVh=opNv?8fmqbRpG;f#?4B#WN#x6?JmUghZO`y-qzxnO-71d;T z;$ov!qnHN@k>FG>pr0jVqdo21>DIb)IFBR}`uriiv9?1^DVEw4l|NlA7ru`3`#WVA z8J%qZn*C$cPny{5#*(K7C-^oRXc+i?b3;Dg#n+;>6r|m&>ABkVn>B6Sw>&&<_movI zKrTFIVc0buKNHK%E}R;okRNLm^ic+5>cJh<=hsv7i1DVNlxl^u{0=faRDmlH1$AMw~7`zq_;1aR6Zu2Yg;D zM|nI<@#xL&l=HPNVTvCI(#a6t7^M5BgNy3I|x@^8KPY0qRf+S{U&==2T= zCm`esEflD6{(W)78-3;!B6UYz_%FOUgjR^?U(B=G#te&|zkSJX{rwqOT=c3*Kcc|& zjB>RfKMs6fPRMNiyakVO_kV|rhx$LWmVC@~5fb91K*I|#r>>+5&ueknXOMWfG2>2B zo_SUGS~6jtS}F(6OR#3OW^2g3TSLRHfh+Cdgare-i0d;;O)E{$b#sXnGjknj>2~chC51&yWNd2OSx8y^qJ2U9W zF@P40sMeG=jOBb4>DP)AR>#EFTFhqWiqee%lkt?92&xtusg3Cc^d#HJI-V)+HJQF!j{Lq8o0)4C^ZM!Xh@l^t_HlS znj(_&pV8=f74*a@?p*>eAvL?(yjsy@kuooWt5!zY17AqO02nA`z|J+t0L*SgOjb`| z5I+%UHvhO74sIT9Iy#?cGk@K5gLTVRe0tF(c z&ydn(LgVai`BB`}UeWhtrW9BC4+U_bqch^2|K~9N@4N^?{OKffQ5;mlBy$NrFSoZL zPrC8LjTlyM%@`%gd?ft@G7fuxsBktoe71TUthg$P)i6B*Uo2et`ZDvab&2qys_QR=~^$W%&DAB81HKN1#OvTuEq89CD$Ntfa5Rg2@LP`oW)CcjCdggdR5|nRIa^h${DSPa)J;a1wIm5k}>_0V!Kv zibnat=RC<+$cm6^DT<1KB0DV;(K*VrHu@A9GwA8$qX>woxstzbK%UF`8XVF}m_GT7 zEex#E|D7JNNF#vxm8`Am0Y8z90>PF1-y{b)bdp0F9Le`$in`}wuzLMe)CAks-gKTj zj|Y@_qf$&G&v8n3^5P-K6n};6bAxP5hJh3P3iTHko6HBeG&Kvxu&bda|^Im|Y{;Z2C69wda^tlL_8du*U$0|ar# zU(NUe$GikNCDMIROot|DaOUv4~K;>-x1HI_+ ztb1R+C}|wHk7gB7&HlQX>e2Yx9%g2t8@U))?aRQi28t2(EZKeQ`?lTCyzSdWuVrt0 zawPBZD}gtR>Gpx?o_jye=9eG%ubQB1B`deS8(94?)aeLb>zDuk1*(pT3h~zr=;ot9 z3zz!1IhZ(>D;Ei8Qj4AXj!h8rYVU8aVaC>p#y*#FWtyt}YxFmTihmXTE0-502Ff_> z+z%0!PyT$%^cMD<*i;Dm#fYm zxpk)A@)=*)d7k0pMBXaq2KJf^c1Ji^S=7*^~8fD;YxC)^x2CM)``u}vGN=Z3WjS)rvV1U_5pY^nR}~yefym+ zLvN8?MY-F2{k2WMIwaA9ueK?yY(O*3&LYPWr@g|v!#B^F?JLuBdA#gbI{&WFL&@xm znlEvSzLo=DH$ex7{-Y``poy1gz}nzp%AgIC`~zZ>Tl@TgX5AHx&4-i!#qK8tRP0LW z_{2GW{REppxbRfV-|Qk#KTL4ptT3BAtFMmf%SU}9#^*8Pn`M3UzsywI5>v0c==61q zg15JeM72I^$0!-J)l-v6D-s9~C~E(8h=%X@Iy&oh+C6L42vrDD>b;5uyem4mc(;>Ak6WGGoE8jpfec7;JWj#ZFA zly4kY0!@{9`$IDWd&T*ai({)Pl6)}w=+M4 z*|<-VqQPUq5A)qxW6yy5)HW~O8gbPMEa#rhw9OYA$jVqSy+rSoxZ)L?b>a&+>3y~K z^US+guTu?@weU{e~tLHK>S)cD%bh7#K}vRQnkZ=SRPDVkE3~q z_NA!1{lQ21crcP;!>2slQ1S3JD|tiwL4aRr8=64PQ?&7NoxKqNDL5Fakovf-G_D~K$;(unNF^`;;O1fx3<}hDZo0xMcmZEQ zNaXp*?WW~$y3l0rSR36ZwBTWD!I_lb&3YK1Xp>yJbPY@;Bgv`8MqJ^a^T)oEP;}L9 z>H&P8Us4tHPidbEj$E)#=8WQs9x25ab|+_c(KCC=0vM+eEb?9@8!;W)z=B%Znp~IDV6BNU5cohnD}s z-1*z!`Rs>@%h^#plV)Ctdk%%xM{$PkB500d|Jrado;dm(cGF7fCf#XG zp^qSfpZWFrgo*2zPuwT4Uu}Jeu^-I=o#nL?P>-{cDcYkaCCSCiaG4$HXMNMK#9G{ z2KKxX`|%9do~#3aD7mz}SqTp)M4j2T)iaB3tBwV4$!Hv$?@Lx!o8SykMHN%()RPCO z`5Cc>wY~W;Wv8fi7PQP|B;@KR^ePK%=m0>jHAwbGW>~)Uu)qVHLBN3SgGl|=eEAvx zpYxvpD?k5pr$7W*$WGxHlaruB9V$u1C;eUp z6p=ByZyxQX=3sm-c2DRy{z&I4^r;INe&Q@7m1mKXNsDTu$7HfRGKvg-HhKP?h5MGN%Sl{mTYrQC^eyF!vi8P_!wk8_sIZblS{Rhd;5R0wq{@^e%=7-xi>;m=b? zbPGgtwF)c{NboS6P1yU z1k_*&t|xFTYykDqOU1Z#(@E|8-m-LZi8x2^AfnrmH1t_-eA&TAvGw9YqA%}Iq~Y67 z!^~#uUzUD?17ZMVFigQygXQ4xa-iJb_Vw&wVwp%u2anSARt$?rJA5a7uVae7Y?v@} zvb9ULrPJk5F-jBjNB#np5@DT2SCbv}^`iG_I&yD<(V`RA9+gC2;Lc)5UgbOeR*cyU zC@#U?Lv{UBuQrkrIo=h{6qPS2iW}O9C%Kgx+jx6}f*ZBDu@?RNu$cjhE0*-E`Qg0h znUM2iTVew362l-maWU@*w&!80Zt^G?DX+6iNLt(TVtDmD0Of&ZSE_~hYYbKGm~23H z*!MNq$UqpT%`^9AU4ea1Wv_cJyIu^1PJ8)u`=<`0me40zoUqQXi?4a|XebaGm-j1r zyFtJ@5$NCOJNYs)h{RcT9eB;a0>4kK_QNz0)wVM8T?5g71}|3n)&{G-u&?jvBZ1HVnsEf7M>SIS}>-por9n zYJx{;+clbXZC1^)&=Mgvs<-)YzCWG%bA(iU+JZYfC`KE#VJT&spx+z#V8kItW)9yB zfPU)LDWwXit&k?uA6r@t#iAhk=_DN`CEuXQViOQhUJZYK;$l6HHEevXA0}F&_qPyz ziGciyPT<(t-77>z_;?gj)TG_Uq4K$0SkgsZK)jx(TITq3iOgcl7^0j={g3hc(GSC< z8S4~z6&dv<6wgc@F9^bMg|`uB=FWKGgi%$|hE{Fq*m#JLu-jssv))tN^!r3jvJZ_f zre&&(#V1YYRK`$3uHMN{hcXSotkpP4SFOc_-)<7f<-pN^8~=~^0xxYS?j(-)NqR9= z-z1%M@cC)i=JsT%Mc>vRP1UUD9i=AG;^ZZHToGKy?m0WrGgui>jwV%JxvE(Pzs%`jT%TvmSib=Q?^jc9_vwaq#4zJ;@X}i5%5fZIIP$Fu06r4b_esMoL zieA#_dyTHD=Q$2IYSpVYemi=u&#omxhPoIe8xNNAm=(NY`5p_7-GJ0AJUJ7km3SxT zd-78xN#_zX=-yH1|IM<%b+V+x_TSF3U6lvFTPy8hLCIH=PZ#@>p9=Hsldf zKQ=0dF$cfIZ5N$(b1wkF1j96VU;Wa| z!Er(q30IOFP5hd{BjFdat(YVIF-C-~q^r9vnopC5kajP6PPv|1)OEY=u|4EeY3ac8 znP)p#K14Dz+VvYKXFTitbo)8=Un@nlw)ay`N3103W%k8~pY1hmat%A zG+uiC5k9E<{)#}f3vFfv@n#weJL;!sc08_+|L3#m^Zbou6>jSeDq$V2&H9+vM$?jp zS&63Knz_``H4{Dt7#$FxvwRbgQ}E9Y}_cz+&XUOO)3=nak+^9|okRoF!eqR~e)W)5#)!P@33 zC#;h;AdgmZsjg9G=km3Fj}K@4bTEbMqeZhUMJod7V##1 z8&Dk>18pN==6Z$R2SfbEHw5{l&uEtW1^ky6H9??WlFbZTP-Ygu7OuoMxANpxc;9qC zZ8F>Q=9?#^mh$l&E~lnd=ic&VPHX5CC_5RG7}|m{D(_m3*H#QHKd4pf2}zG)^=na} zF-rrV6K1p}_B0QW_xT?IM?fMPvdjt@SsrOM40VH3P=k4}Kokp6C^=?}6kZk-hoWgo z(ayJdFTZ3ru~RD-)yWh8WgTWX_I?R`SWvIBjSWu*mB_AzUu(TEqnPd|a=yU;k;>$Z zP!VaGD&W?OfrY~{+~ItVL3obsNUn_Sfmhm6iqD+8BP(8A;=v@2OoZfvzl-m@-d!c6 z-BgIOJy*BKY6j2ctX+(~=X!d`#kZJgH1iAhJeqVg`_NbRX%xS%>5QqI#;^u5AP^1J z8;Y*TERG(%DNPEBQ8fOOXesfX3E$9_gdvB3JIaX)n~{iI3F?lJ&F=lZ?OA-Dz@bo_ zK77ro8?HZPU7QON_+P-bg)68_s@ zPo8fm9-db)$6Jwc9~9N2`WjIecg~JO;ACdg#;5|5&$z|wQ>)4kM3O~cPAA(;T~Z=) z-q2qFv;*1dANn)1PGh-+G?jY4yVv447M2CFt%PLjveOG6NnM7w-VFMh|$coYY^{N5%TSNOdvPeCt^lHbMMj5C+D%*k7S;PkxH6 zpQSQ}PH(^GCGYTBHW;0Fj0$#qGoTl4f1myvV^hQ1t>IcIo;&m=+srtz%WkB_jSnYf z*WTCbYm72_vjROeO*0X(k!HxS`3BmfW@uy_wF$bF2Ko^N-E5B3QzLpPjRX@+iild2 z2qzzeO`v%~o7iar>--q$f)?2PpdooT3*q)B@1f+cGB=5;cK|@u8yuC_Y%?QY-b7>f zq@ZLF3Mfgh*oNcBX};o%d(X^6>dOf0xHR@S;bv&d$80qHuC6UT{1@C@IVzKJ`!-yp zmj=Sx#KX7P;u*Rk?*Y<%xyWhmalQ53fsZ1Bt#~1qM(MY;7hmaL?e_jTTFqg!~*kuii_0Mhkqm zka02_r0a5<0uj66R!=8siXxhd{J3`1t`p9S=dL-nb-6XeOpXXp)Bb=p6^-0kY-xz1 z6wz*W-X%i+nAzV;yai$*M~qe$2)VMx_8FBK2d9InMOw`&f;am9j&`;D{P2;evb$A1 zjD}iLHNuBe^<6H^9=T|1v$1J*h0n%4*^loHdOQvfMP`piqs(6A-F@8s(bi3Ue1lnwlu z4)}k#ddsk;|EPbQl%ceAZFGs0bc|-SgdkEcg;&@AU+GHfq`5O)JB?BeTr3c<&l^IP3p5?BUqx z3KI$8^)~NG@7fI#C>EEgFDg2Va=C~&R@?>_kO(DKg-;AC~(bO z=gzcW-0z-7kc^nQ;ly z^!iANa!b@~^d4jk_FyN|t8_Tvj>y~LorH5Cae3@1VJ38&dSvdG=kb{x$mTCTdAL+$ z^hrk^;k;Kw5_NmbO`K|mN$ZTARI0exUBOPp-!4SN337rt6uhRl0wC!c${!cb`{!L& z|1Dqcad)ScjS$i7jZqgJYu-H0CL$OTGAwo;d6lz({h%va7{51~ zukknBwD`$<&AUUR?nWE7csv-V>VH*}{$JHxFL9vD$&hMRZL!U)=%Xuv>=dwgiNRZS zcvI5|Plaai5B2s8WDg;%&O*OMWqSQK)nf77aK~bXlV@*wON9};A|~JRA$p9oo-gQT zi5=Z;lP|BB7Bm9GJ(_Jk8lIo|aJtGY>(jee*7}%9+M`&mu9`Uk{IL8cvAp4`-?9(^Nz8W(!)zhqvwxzY!rw5 zHbef>znsVUQncuHMFJTa6oh?cO)W|F3HI9OwZ4S570NxsEi(rE4sKFSxC8j}&xgP7 z=gQOCC3Jxa3rQ-e2`P3Bq#ej2BfJFNT|Wf`*&P~%R^fSs$xd|6b zQ)GXJSI}$hue3N1ym~zYG8rWduVBi=()Rn>U#c;5tFXuzOmgd;8%)~$C=ZosU@!(B zR?`b2@NBe?ot2B_gVd2LSqJj@vPFJ3DHx5N^$kh0zUSLK6xSw<2l@~ksRawV`8^0m zF5C{Dh+p?GA8d?u`dt1mn+SkhFV?3I1RyB3XGnrIu8rhtptlsCSF_2(@prUXp(HKL z--dKrD`qpo8UTLpu=zZDzsZ&Hw>lZ{EU0eI@TtOQDtuZZU<%Tc&IKV2F;#4TZ}{$J zEAggG&9#cE2e5ZmO1S2 zgB%M;WGJfmpM!HJ~@8T{@7GYeaNYg z=l`L}F$6ak+E0$((L^0hz3M|xr-;H8c!;mq{Nsa`IOy~@djjEZ9fcxYHBOib>||5{4qSkIHkK z2n3cS?nw~4Yz{Y+cpH^#o}vnb_(onc8smi2t#rwv9;B0CJr&6@Hr2|y9GXy>fBZ_t z%UEc4)wPYdju%^T|LOAUvcU0I=PTJmjP=PGyRn!8LaOOh5|DDMr|FZna^o#L0hGkf^+{m$bWl4n?sM+OEX|VKd+#2 z6ZwRZFRd#MoSu%vtT-D#qDqI4VD{6L}Q*`Wf^KQQN>~)Xdy}` zcDXWrpn>4WKBpg3hL5_Ib!osq90FK4n(5!aAY#;-MFH9+xW`=rPd^Bt z)bw#0O`j~1r^)#@nE4%bp@1`D->;0`I6(>HNONd$EqzmvQ*n4%c;ONB5NzvR_D9=z z#^CCy7~M?x`<5y%clU;g&LQ~wXy8(VVX)6`^@BuW>>$0%*=E98|`Z{*o1MiE&B%*n?3!JGl%U!=;0PV1=&WA7_}gs>1$ZA++Qtw(cV zuMuW!Jh2SdL0W{TmQ(M-(rB49zPuFD7Aq@G&@-pYeLUD^rtL<;s^NXV*I|<2(mUzT z$9#XD!;fMH@1Td2nb*-ggEVDdR&*Z9#?^TapZ8R0XT2hx7-`Vy%<5QGOnw?hbNgIK zk2VcEYt@@wu8wO>v#Cx%Wcsrc_?|nhb_1Bho%OPc-kSxj0Q(Y?CejgxBzImYypo?4 z(k6@}$oJ=>iMbkI-a$7@_xmd~{y&KVD(_jaU%KaTTpN?G3TpjAAUnQGf;WtV3LrZt za#CY9zBkDAA;iz5&Cj^qT05S}C%8G|!HDJa`OxLBVO=L!I@dX4goxLW48jV-Z73?^|1{wxZsu=`c>WsM z=d^sh%!WA>v|~1T$~s6&Ve)sO5$3Z7dt@w(hn)bnVU9zAsndzC@1qq0i4g%<>3_)~ zwTloF=@7){0W#$UF-Dx%VL(}p|BaUnHi&5hEI`O{O@ZX2^`88>q@+?^9AqmZCa6-* z)Wx+z^nS>&M~qGz!CUun5!MhL*IPTr<6X&Vxk`NZx{2cnu^G%u>-AG6~9n)B~ULwopObfpZQw=?*$ilo#tKi zPfFwGU1EDL$m!0z9??OaY8G5bV#V8@a}bgZZav2E&9dzN+p37YhaAkJR(N4YB$xSe z`sp)rn!I^8?)spgZ`%CcWsN;rsFl2V;H~LqWv;-rv*hou%AhvnlU;ICH~WdgOe5f> z*e~Pp@aMsxRac;<+Q&T*a^#icd4Qbp#7k$mfA_!cBi=vVi@0}js~Xq8roAV+70Lmf zTb-WO^2#pmMF>psr$7h&Sb{LUIB4e#HmLqTPN0w|x94E80_qA`DxzgTh!RJ@NOc0!|`qOHkZ;|ZMfps8_}E_ zc%DpfD!)JZtaGOktMt3HW{V%i3)U*1)U2+H5YlJ81ngOS&I#a!-B7g~5**u@b}V(m+LV^(ZbJDcYllDLITMkO!6fuv4$ zQGN2fm{9-+8{F_GB32sFmy7#(jHO_dE%u|=AcfeQ#;UK{&7-FWT3o>Q=sv;9Xr-ua&gRdil|Hr06@v6hvYSanct$hGCW)*! z@qAh>+@#=~YbRr)f`#jF$!-&i7*lD$`SB|C-u!c2RBg36T-fXiIb}T&lUBjp+5sGKk zsOYgH34DB(MXdaPch0^`^UJ%da^l~yfN_0@^|*h2>01Tt=%aBtF3r!Fe*!WO32FNf ziT2USLsWJRT8}%9m*#HWW=rBV)6!uJwLbtPguK_55`82d2XT*PfiJE|CVOBO7vDcv1tuDKhe=6P@Tu6fmi76OFsZ@h9 z&u24ud#ubuVewYsw9XtcO@^AIx%O@KcONK_r8$s*}~;d~HW8vM{fINh1?HXzN4QMSeYVI8>Y2Z z;-T|r?x*sulrVPQaMgc{PW#1&C=m8~Ud3aB1lBhWQ@*%7S?Va55OMBm_zkX>R#@co zYa`~o7u6vkhHg$iY|%F39NeRz#~hg-#%y{{xQjIqSn4^TxNU*O)W6f}e;FtFn_e5c zg77Hs^gK>hfqt%MWQ5j8HFk8lt3o`dj4mBI@V2J4@fK`19qNo3m-%uS=q5Tq4p~an zG)Cc)gMIds!07nkUc~1&D)6c|e>C71s|2q{A*{|9HA^#HQ3||}pB<`zG!Nh(B_esP z0Q8;`c~KzPvlWp5j>hy}opKbBLBayD=x+8I?}JV6eb$Ll8qN+kTQL_jsBy#C~AQtm%bb`s@uCTt<*7B6@Y z<-Ce)*)G`xxQtysWD*`*jlR}dW0RIPb&(Gm)f|zN+Huk$JBue0iuILRYNjC$GdA5| z6SG>l++L9>v&>q4J3}JTN*58&)G@1bQd8MnlIzO-yWUSlyOXu0lWEWDoUqy9CvG;5hV7RysRPZcWUQ~$8V_-l0t8#{UnR+}uI-NHCocEWnpo&H zsfczcpFZpVZL-`Okln~oLd*4Pa!8->-FHO=^@lh^%)yGsB|?C z$5!T56uDR$fxhh)vjmRg`N@T+X^{}wbG}9TfKfOKZc;=z;)EpV zLwsag^^R(Cl8)T97BQH9oB${?DeoW;p_Y+Zb!m^H$7gX7QjFL^`5A!p@=i)pBKm&l zQ5+ceJ@xBJ-qKp$Jhk*ao7oDxi+Ei|`-?(Xde8xE`@P4)*vT za!CC9q&=?TWK+(O1jYT-J)>fm z`$$PR>Li)($|mWN@09p8Ro9Zsvh=tYrX^Req$G(iU_4F)%r$w3iKrZEhR$^eRKlt= z7@@gSJS#P=5!;PKm((RUofNM(e+#q39BQgRVZ>LsSvxg?TphT$mMfggl zZ1P0BuQ=`c$)$#`U1X0VM@vXaeS@(gh=8r*!LwhMPU}Gz1bCojQVkF`sm|&r_0b6c zo|ymk9;w>&7R}f4!4kY8usEdytf$`|qT72B$=P|rzz7Uum#O6-MP7fr%G~HBeE_?E zc$$qhtN#|InDj_Umxa`u{nFf^2o(8EPE{2SEYiW`5qe~nPyW189;$Ji)-fu6{x!$h zdP(UCdd$`y5HHxdutD!N=lyQF9%>rF2zn>T_uCz9G?DX zC5TQYf5iHk{d9YnQdLP7Z?EN7rm|84G;YZ&5Hb~b2)$Tbho(R{532K>HBdOfN1*Xp z#q&m6QN@`t&mp<45zvJ{%?AOZE1gxheqrK3gxn>pJDQH|ebbTZ4=1O_b0r*Pa^w4_ zC^kz#_XYu~rvRT*ex$wpdGSPfsU%NRV+3!&Expa|@6gr^yl^`%4BiT}>DR~adL1I_ ziYBszPdp7^MeHPBXky>irm|FlU4PM|GHzOI4eedh9UA%xlfdtKPM=2~a%fIPO{Bm)lJZy|wMsofni?1Xog z4QLP`AtAd{dD~2c3g7O34aBuy?f#STZy^E9#PAm)a6+G6)r*KIYE3nW|F@9hrwsR- zInVsI!C1c6;5!7Gbi@I$Ty|x_x6j|SshgtMj4SR1gTfOr+&nz-!&%Y;tv5n%3w<}qb;u!_0Aw5fp%%vji{ z?`-mGI75WecofmVlSM)F%OIx(RBTeIssp`t^FFAa9`}4mSk+YQ7S$x&O$NdDhCp48 zWpI8o`X`r+fj>l>W(S851c?+IWT`zy$D$u?Rtu86d#H!iDOAjXV%9TGg~#B5LU^Mg zOzV&2&|hL*59W5e>QD!M4J#&1Vw6JOwakH^*ymMcyXyjCSBD*N-5chsoe^$p28QmJ zR8B|M2Y>!F+5DRHC^i`u$xS*~d7Ias6G-?fwl~*~5BcewGZ1r;RhCMX@e)Q=5`d+5 zU-{}n%P^43=2XNoisHG{Y-NUrmkMG)zoYL=dGQbT+3M>fiK_69W?t-EhToA--!qfO z{2OdGXdP? zxx_~WWuR!PK@xaAN44eYpKVMx%;xHeYZj>QALgvEA|JVezKH{illtM-RY2z(3^gZ2UD=fMy__rKqND+g4Hb3*UQoz82&muw8IK(ki8eBN07`c z6c+;`i@{-G-=5$m+e1$G?Tg#^od2we{UJ4~bkPO1^!T|GhMT!``fp2j;s?q7UtS&# z&!j(mU5!@w@^C3N1UDOb514r$QEfC4`V3OgGo#6O7Q`v4+Ch6%?h~lZ<5%}4nX6+w z^hg9$DK14(>y4EarLjo8?&GQU~Y zJR8lc74RM4qsYe2AB>bRq1(f@L2>^~(N@SCFJejgw^k?j@+8)HbU(K0Ww#GdGk+3& zbI7`5819!`iboRIzz(gePq@;ob@frzElj1H-;&wW0W%+3$Mz@$O03 z!}{+Q&oc*$@I3K*`3t2ExSLMTdAM{w4kG}N_Q?7$@f&wO$Q652Otb|8DuWs8Xk-Ie5RjS;o!KWC2Bc4S0v+nAEWp=esMU$b|So zs>i4f=XB&e@P5XFKgE1IvZML!zyAyw$#E{rYG*k#R+>9j8-T?V&FJ9{xSSyPz}&)) z|D};5II8XLLKPJAFF4cpZ+*L{FeD`&%lmA~5m>MTFums>GESxQgfGX#AmC=^D0VCj z;#sbU8ELA%ONcge;0AhV2mQ+YFk6~&*w203AS{-yReGbiGt=E^+g?lR_l}G`coA9P z(3xnTQxhYX6?N)n*SyCYr;wu-kk1Fg!-W_o8w$LGBF`9)Jl-|lu6&sm&zAmOqm)<1D zzxQJi7&hO3kC|mVk0&<=5?t;c9irdm#*X5j??5}R{b!Xy3kSl$3UyD8+0q?AUF1?4 z#rjji&K);v*9()9p7rw>7#U06cbx8cs~q&iogF44&-t{b!Zw%^(IW8QO3n8;rC6+_ z#_>;rFF2*5Z0l!iu28K8$mc)NuYMFTrRYbsn##%pvENgsD8Oa8!zcd=e~OX*}kZJr_1@_ z##XW;vLjxtT_zUwsRKk-&CqNXDd4ul-R;$sToX-KU@Y)ERpmTaBrwL>_MQXVDMh13 zY1qB;8@BqxP?gs%9Ttsf&5DErjj9YBh%OlIwS=i<9#x#c;k*-rA|$ z2z>Da5f8T8*wBAr#L0Bd%zAK6|MJfdO!hl_?R-W-d-<1SFdEeQUc!bj#hrU<`^Ah# z_MW@-I4ks~(mFd+{OZ|lz36Xp*@toenpCo{Soeb|erYtb$d6}W>Zp1wlG8kOJX%8%iSja26)Cgca-_1wOVZKw7 zd+&H&^$D2+m(%ujbrd=7)WnCD5B35uUv^@#gh>BUG5Vi!$F@~$WDdt5XR_Ypa6OXB z0i7n`)mxRZS}BL~6)MU1A*2uEBkczMfD$O(72r=z`kG0Bg-G?3deQxJ2)*cJL*tzh zpit>fXVb_>1Aald zrWrpaC)=?~Y`cjg!!Fy8?8e9#-8PCU=jyif)Y?Cd(dzvxiJak4-k9mL8jziPr-sCfZ zLz&nJ9iOR~X!VsC^!`dr*Sp?CAq%*MveX4nJ}8ZcUF_w!+C)o&$#HAy105MND(6^o zLGj%hWC)lOrbhL7D~a+)-u!{o4Y27zbLY<{b8#DD1VchBmV2 z_A)EADwC~m#)h+0PXH9>qXYj}SF|20t@COyjeObEYY2zYSCkWQQX+dcE@T)i-v!iB z$H<-i)N|}(b~82Ot>JC*;{?U|x=U2e>xE;X7mZXzoL34`q4A#7kB@85E)PlLI;E@6 z<`s$tpSS1$f8Hzt+E}-X+ThlkP$O8s5|4 zOvoVrc;(KS%3njhU3ww5r!g4$*x2q=%euvQ@Kiv8uYADZ5h$)vA>fG{UU*mcA`?#G zo%pU$ajArX8!y{umq+rTY68d3U^IpW8}(b;nD=N53lGK&ij#?Bt_M~k8Hx}*P@dv8 z#;8Y12?qWsax$XY;0>^3yPT=|$LG;Jx`Cfj<~YZ`>R9+wY#P=GC>`s}7b z3LDs+->IrO;AeYRp||Mr$sf}S2R-XCY*h;ojA(jo-}f2(kyDLiL3hZ|{7Kx|iy*~3 zVTeCE@Y2A0q;2{OU*iNF$9Ot^9-YNiHMn6=@8D*+v9l_C5FiJDa2rZAFbJ=Sq#=8f^e$dn z#WE$>#(~N4@MB8eR^{1}HgL!_e$@7%1I3NV;Cy=kihFz-h9$Y*#UuRpiV6T$<>5p= z?-D5A^B87^WkBPp4t-OX{L1N3OR6y)Mk~0BpY|d)J|GB5|_1+qXCzRXHb*oT*Y_T>N ztI(fvY|7Y~?aoRnb2s8JS7p8}AqTyA7XwK5VDmLjj?HD4GUaavsJ%}@2Tr$(f^gUs zdg6^`NHp{0<=v2iICyw!B$sUw0eCFrI5gtwq!%rx4txp3|GdLic#Oj&WT&!o781x< zMk3ya6B5~&srX4AQ%)+7%6@q+vn83e-TnG6-zjsNdv=jQ7CXpg_3`0>lKnt=$}+3DBqGh`kn- zs}dDh(<6OCQn551-SAYlEsb}u+;H+!4OgD8GSifo~&1yg%`Vf%D(CLwWXZWMT0S%3% z?sK9{U9|K+P0sY*Oc%E6PpaEg2S7hR&%U|a1b}~H&%i)%23`eV)$5jn+$HJpddH4d z6Ml@3A$uA-uliY-g02p+*r2R^q??x)XA4TZNtl&9pp7{EMf7eX2F@HFb9u8_7N)X(-FJehs7z)QA)6e1hD_&MmzT`>X*ADmr?-N*%i8;0c}t|;0dr@;ed+_ z0MRGmo#+!U@!?N?93V>^y3x(>K8cFYsQ2G$=f>g3TVJPVf6z^-mT(g(_g(g8pVObN zi>C@!JfOI+TOUFg!i8Wp&7m2S3osEU5nk>mDo5_;kr9~9qT-iD;p*Wek==??85Lmn3Nv^2PX756NW zTKjVvxg&iz*~i#vHS7bW&wUfspJ|#-!g)VB=kZ?B$2cMDSQML>2S``ZAEdh}Y>4|v z<=8oz3>V%$cRHH$Xb|)v==}DoiQwC16{#0bAad0)helkw_=4G@z+o)PBXlE~i_kN+OExcDiyZ^AC;XX>6E`qkBjP;rKGuV5S``o1$-p`&i{8NUgV;DEn=51Z%eRZGUw3A3_* z7WV+mb)qJ=ZFA`J#<28Z5}@DGt!YzlfnWRZ@bV@|x^DKyO5bBE zT!<2A;a=~KN$g?veY?lt%+?x-1qf0~n|sM1Xmmo(AyeQdfR{BBAP8sv1Ry?MU$aYk ztmy3PA{Ye$WqAdOv!-7`qlN!gHn4#)9w_HGwKS>!I2Z~o8176{#V)v7kmz}q+3{j< zkulxY|KJr9$!7<`vXWe@YpKOZ6bV;jtWSsLZ-hr{KR+gNu!e%L}&#Y z`&EF%3TU$tOn)vYaOAH9(@J_=yq)a$imBOcr%fo8V?rozX|sc^m1hAYMV)NHN$Bi| z&Xn%qYzKVvhFcBUlTx8Sblc7yS9(R9I4nz2F_Gv4Mt9So))u;MGDeu|5z*Uo!~bhs z0eaL?S3?&nEM4FNsdmVDfXwkF)qR2N;nTDq*l=As1KX@;`k}$G&Ak22mOlPZ$G;>1 znp0UWsN5~awSFy<57|iy1*U2k%I1FTYy}4$vS<9Y{t+72L#6O-MWQX(EpfSYqFyWO@b(1Ka1lZ@q3>;$C?vda_$(QQm`*?;!!(_OPVx+TZFoTR$jj}<} zAA#tFxN@7G%L79r3*Q6L)p#?2pm5+Ff?ZXs2Q2uW;sd`84riwLx|J?<900&uetr`6 za%2qG3OK)2KM9_PG5dyqF?@xq=*k^salCMJT)uI$;A~94>q*dL&qaQt`k zrynrw&N-@lIJY733u{b&sXVBz1!zDBl0V}=`udfBZqU6rkNM^jBw`qUc{^qjc*xHv zGVDWQB~bTsB3t7*Yv$z_Z>=29?;MC`RQ6_t#JNNqm|K+wX_$1xn1s4dNJ4>QF%iea ztP|b460Rsu$hSdgT(8&K9(y@RZm}U}$X;I8B8R(snk!o;Ts_X|FIMM(7moilwG01t zlwPAEfb%FP3{ezGhYDtgNe*}9^27D6h_E|s4>Epb)#T<#akt{PUhs9M_6W;_mxg2Q zjEkW0!r7o0u1((8p?CgU-FYV4wZ6iUyobbW|0lB8^m9u9k3hE66pR+L5X{Z5Z07;H zl*2bq+4cX{|0w}sWU%ZRZ+>7On%o}QGg)Y`QTFk=$nRe}!BMl2NEQY{Pj0K}aBd?X zz(@yHea)&3@Dek9DD-&s8)r7m)2JSc?|^vuCkrcdSO*aNy)3=>$5($<`JO88HhaeM z-d{6MG@0RRd4cHOI_qw!SuFjI6r~UQe>Nm;c*j0DLS;S%`_&ICm%MXoW`-r{D2ANt zeCk7+Rf?L9m$$kcNbxYczrUE_1Ewj9Mn-~?`JdKhED_+~L22E)0RJ|O z{|hqK0SEb)?>2-H zJPQjT!TCLaoj{6WL-bg)@JRK*0t9J5gmGqZY$kP|Wgy*FrE;!b>wm~~)FgR*-cqy) z4Lh&v*kfTIhs!HJ*x)A94)9uKu%(evtLK6_zc1AdC~#feUKbrN(>ZL4`1$1t;*T&m z@|nu-c=Kc}o%Z@VJqp>$p76GU<^pZ>=4=*#GX~($dUiT4= z4FE2)0UU;ku+Qj?ECXsF?O!IfuZB_1K<9z-Bx^6|r>+Astoq{)O*oeKQ`4`qvB^)j z*!E%F3)8_ET51LOlWx)~T6FnxtTt|APhXU=V^w4f#$iIxax+_&-6|eT9exQ57&J5e zsXhXH0n8cf1_~^+6Y1R*q~)GB5S9&{Ib(LSlwrC}9{6+}+hjH7g>={)(6nQl>WG{B z9D$fG92@`;>lD)Z`~TOw+~mdH&OG%ZO5q1g$%*QToU=a~0mayNQ?Y>p zzF~s;RTbZWliq>$2fx*@Tg)gSyzHd5Fp|tQcG`){|b$F3}|xtS)bIU=}91KK^8LGN@dt}af1-`3P}?0n$CMw zaPNKxb#j1zo4Puc^~>r#0`+ob=K-m3^L6)C}-!z%R*63zvFT0_zQl4>vvGeADV)(I8|cxjY>e| z_xLAFT7yS?rra?H6quHJj$wehP}rkStA6wIIc5XMI+`i!hWY)~@L`P;`tKd&)JlWc zbVtXFFZ+qKh+E(9Mf7vzz&5JXIz6bXQ;TuSNNXQHyF$AbUaYbFVV+w*hscxYPHOOs zHNG+^WM&a{Mf;ey^2F{jd0Inf8=vWE>_^z?C4dLf<%B&o6;)120?WWdbMJyVd0RgR zM8*v5vfXM@O9{PbpW3!i;fbpFqJ0<)^1j{D7Tp9zREahWV_2d|SXd~SIIJTmhOH(E z33I0KXX!pR89hPAS1XC|G+{bI0IB|#eo%N~1^$ET*%LU~#lQHv|56F4?<^G5x6x)< zAp|3H{QUfX7QlnGLT_h&CH*}Gav&2Xbfo#ei`mYtMPt$VZ=gT;h;-meLV=7foRD}3 zJM@0qLp*;;+S(q=cr2?Ef~_{oQsppJnKq?T9XGQl8rT+291_u<^Ah#y0nfRlV)cbv zG2&{E=)~*71UDU|*M^x}3vjcE0=7P$`yJgdp6*T%lBz*1P9|Ij-w3)_B{{TnAM4BW z+P36hctZ z$Ngy5+;oSCQ{L#NLggM*T)oBOS@HH0wVUW8iK#-ks}t;n z2ZPJSb|^}dB4Hk-Eb1;~*9sfy%0@Z4-Fle+mdQX>CbpzcKc&( zwI}op?Ufrm;QHWV-3aCv@Zw8>`S|2((#X=aKM_&Gc}5h(5OP@P2g}LV7E6_Si_evA zB|_;V$az)3QC%im6Zf}mfu1jBB19kcSjq2$HnU?_0V$EfaIZlk&*bj>_lmf%ObQ6C zfiNDD;Nce$pKoe8%dlDtg3o<~P0Xh#F4S;`-Oc^7+Z!y&CIiR+b&CR4L88~o`ZprE z?^g!1>MngHsQ@(^-Dq(z#_^E6CjH-4EBsRNT^e-QGT-~165aCoTN?B#TS7~66_@;Y zI_71^5o&02C=nO6PKJ3S)(9yTpYT0w=ECfcfM0e!%k4uL$+A1%My>0Umwo7Gg~}-L zZ8dWyA1{p>ajFD7n*U{TPZp}aS@L+SIUaMdtd47Y&UAm>Us*jd=YP`+fGW1pQ%XPW zQBY@tGUuR!GtYC+L8R`a{(zR?81zgyu8 z7!auk*gF1)iutHo@~#g7+{iUJf2Oa`+SS!n0l>aL-sb4a9EIG$cB+W(TQ{>p1E3fe z3V}VvHvswaTNnj10NJ3Ss3Ysgab}xX*hlT#q^@|>nH%gX+qCBvZhj0Exc>iA9HD>3 z=tD?Oc=14i^s~&CkQ0UXu%AoMo$CIqpRQ}AK z-;PUo|2R+|frw$9nQvwJqR^T|VrD(c9)c7h!m7iHuMeK#;MEBAgt2H59Vq~}EbKPy zAo8??`YgWZl3HiGvakd7$*y>CgyPhW_7j}3!$l>jD#hFxxd;ofch?F)F2RYc8ogcP zan?FK4LdV#3a@wbRKI;ZD>41=)PVv{SB-zo zq2GER#j(B+*eo}qs`wD>`NUN_@y)jMe~QU6EnWcQVp)~!MYwNnHx{R`Lmz9rK!yQ( zYYTvFSf1o{XpDc~K4TCew63KIlb>S+oM~15ZtcQ=Sr-)KK9DLvqv~JRi})==ES1rk zC1RR9fp#J!NT811hP1a!XDv!~i*-*DmU`-pe{@@z>$$}6d#(o5MXv5f&3ouc>gz01?)A%TKm!Sm zKPkg_pE;t2Bj3BR9sd0oy1&G&eZv4K(14%=2II7_xKgB=g-sv5i`v3}aT&?R7V!X0 zEY%`z#oLR>;Z)yQZecvqC$3srYw@04cJAWR)M5V_gvsaF5q;oO!#XVP8FX^tJpG|k(u znSZCjEVz769kK047a{zn>;CcbW(xnqt6NIVSmnRLl2BOMj10>$&{Bha+_dlSJbbN| zyCb>#-G*GPr7|tuzndm&IzrP#Pw5} zNX9&`#FB(|haj5uQMhVT-E`Wgr~!ci>BFWxHA{5bZ+*b-Fn5wt?0?HRF!*A?@LmjN z=2VdIK(cA-im?ATc!MjM9vt2DGIku7R7P;MZmJ5aRr40$JjaWSck_xlb+aGI`kcgdbaZ5BY0t~WU}*i{ z|FJ1cJI+&gxdB_$JLe@uDNz`Fv@k){B!l}xNjI z22b(m2;SAM4R8)66<+&3$T-H*9|&qK?s|uUo!hJ%8xGKo#e(vpkX{R_s0v_m-cNNg5%hjyy1Y7QGXkF1G@OdO+)4R?@MIRxZf*FT3I8sc2_bv- z+z;Wf^Q7@)D`m*nPBtI0{r( zLD~xk9ilvzTemET`}xz&^xm`a{b|z`pu=&@n;7l035{viCQ}`SUCbYmHae; zn7UqI(~fOu6+z{Z0A4;%l`!Uy#Se|u@zLemY=xPpBfrvz8evAuMV$wD)4U={n2UQ6 z!vwP8>M8FRia`mOG%r-(c!M51h`glv67YVQMs;_5TkqpNjle*D6vmcrIUATaxT#OA zFYZe58J8vn1qBU8V*nY-i}gAipj7}K5h@g}H^3Ep1V&+IcBRLNfuZBh^rs)@|Dwzf z2+i*Us_k)>v?s}++~1a~cP6#(vKBsc5(k#^WD(aNwp@jPlYoeWA8t#kD^9J*1kICF z3HwYv*)6q|7+qZ4FcK!-+C`7g`?bq(!g8?$40mr|LT&6G@O2J>93)MN;=~`C@5;du zH7JZ8@bSS!(a!Thq#H+I=;`Y6cLfn|Qg0y2YE>!WEP2}=ld2PpEH)2P!!yE_GG~Jt z4JAkF{v`oA1daOC%S}vgQB9ZSo)?heVt0A6(=k%yoXm}4oAmfdcq%9UnCj@hu@M{e z3=8dS*Yw6PJ7>tWD*RB5I$i7ki8{M}43K<6twvnqYQvkZ4}>Kzmvb&g;D<&kDD-RG zYmZz$ah*%mo6uDLj|Z)dW(Z@>6coy(Z+KW|c1ph)e$t(eCC9~839r*DcdMsl+X@BW z*>?M*IPfS~>AnLBoYKR)fa|(7>%KP=ymSMNKw>IMpu`s-9l_onq0w6q8!vBDwax5Qu2~ zKQP*(;J7+dF-+c^E$%V0=l>YH_qX9j<{K^V)X)dY#<@)EcT`M z*1iH*HiB$Q-vTNGP;Z>Xz(ZXb7R!&<)eG*^=FUUKW~c~59pA~E#K%ZAp;&i1S)VGv z6{`BUZUyJsDy%ia$&h!e`LW+D#jZ=$zvRCgMaf;lwB&Uun3+m%8#-u3Yn1~TI6g&- zQC}NwhF&#xnYLfPG;LX5k=>}c4nDR~UH&Cikd%}JV!F}(ELMW+bX2rLcz@`#cz&rc@ zh6GqL=?3Wz>5y&-K{^DK?i_l60p1_aIq&&@ zb1hxKl0AFh_mxvjt2`o|t9}=%8REXpJR0P2G5ZTOrb>ar1RR(E?Lzc8y`0h}l~+(8 zO>^N<_}g}o5|QZ`6<1}YrmNXXq2#*8xLttW)DMtgQHko8aOxLM3ZFB+Tb0WAQ3EL3 zHQ6m;w8w}Pw+SE5HzT~k!`-Cn7purLk^#}5wj$#cShRrkgUXYNz4S!uCd@nAJZ1d9 zge|FeBb1cjU@1=TzjJ-zSeCOdw(vd|B8R|j7FFwUTR5#MyKi@>Vp^VVx}Ug;i;I2y zegpr{MgI5Ky!&PFJGpWrSPnPUfrycG_B2F&BYOMLV(%i!^4KZ~Qew?Pt?VToXsZ+N zW8Hw~sRihN_WrdgN<7G%cOlKPo`;651KRVjA&=66c(`7(ea7X{;wzF-UHuE-0-R?| zy*Lk1#@68#8kAof#@o_buQPmk&Nzs+H}HRHNfI`rFHJi1#wG6UZ71`Bzw6aWKhr6T zEiQ$Z2Xo3MUh61}4nO{dSk*7}n+6^Ou|C0#Tr@F0ms1McK=l3_g_)UOJ1P&%YHw)~ z-4@`t6M*Dzdv#Vc?lI ziI4_6Hm&r7tw&OR{(UM8H?hKsqYKzCPP4zzJn@!P4!>qPq|6( zt$JhP9Cz*Qu`=Nbx68eqs8nAr8Gh#)zpCcG-RENZJknpJwSKbsbo=f!Hb69qJsN-{ zdciIw&%cf?zt+pWDaPktN7B|%*ZoQ5V>UaI?Mj(^z|o(Vp*vnSJzA*#()R z=t+xtUh^0?1c3A#-4Exv-g*{S@_zd363PE@_JvvOru)z0>SKx1jtl-8rO{NbLxZLn zG-TjHefGITG=9{+v91Alt}nxJ6KU1j-+`odvdH`6^h~JFO%DxX9Rt4VU*3%iKOx8o zgz3t3vhW7_YkJZ3HtWa334qXKQ#Hrm;K2%=;wpOOyHzN?$0y^#vDqZLGtw$=fEB(IduP5Y0Q^#)WZUesq6_XBw zz_9xKMx#1Fb%492kk1II=+I@L0v^G&F39~+L)iAi*MX)I5x0Q&#v@>;Q0;$n{7v*z z^<+(d(Qbwodn%msr}+sU_}StA>nqc2t(j(X`4$>sweENRCx2Yv1JBmdE=P^%@tiw< zfH}fH?>5Ty<5^p>4GexY%~%1T-6x^jua$HuKtrnj`NhTd7mjN5M15O3ETW}IY5LZK zipptEy#4n=h2K%}XgPs^ZE2D8Z=M2tQ@5`s#2DL8O>Rc00c0MqR{|kP(cH^pw}2bB z(f8Y|XlUVQ(L^xW(af6@iZNu4=S#n&1x?(r;gS~M5%zr6XyJfmr*OzI?L__ z=c0>%yzskka|jYDHvVIQuVJ_bYqUfibK}Ar!ULWGfthQy=d6rxzu4XJ{Feo0(EqwK zK<=9CSe``pdy2$E-B!M_1?FCbcCRo-UMIiu9rX0egp6o^kPjM~MWNqC!@*7D;zeq~ zkz$6(|KmuZ3e*_+4x9$nulU}s3Bc$v=;qds8%#$JESPlSF~o~~=G9)8Q;+wjrawb& zR(#dd4MX4D*e}+ah00_6@nkKHI%o@@@s7SkOVam3}%?xS*iLGg8Q|hCWjHo zEgRCq$6E*rsooG)ntJ?H?olLp05;#tpgdlperW z12-KDlVc4~vci7wWA6Eb>KY&Blhs^hyZqYZ_SjcbcvsF;;f30He{K{eWP($_UiJ+> z%R3f&SAZ8(_@Y&5&3@}LntbazGa#&KqOpIPk^8g0UOGTDie%>n#(JEkv#%`CM&zsi zraC=f{E}B>##-L<@KP~;%WzM@ya3$hHbgUBG^zm>r?YC9-1lzh43PQiq!01j|BVD3 z+^OGpM2p0m9g%Q@@4}wX@XXB0FB2>*tZm?iNqRLd)Sp%cd|1{96sYNIx*vU%hzcP< zPQ`2aO4wk#k`emYH$nC1r-p0OMMT&I)57$>HQ3z!3{`Tzyz-WeLkF`skg`MyBB+b-3< zRlSqb1*&)bQOK6utDNg=cki7_@WWNENJm=MUkB^%>^CIxqGP#`?j<(TktKjQ+}QZA z=r%DM*jxbK?LorB9wRXFxWWcN&^XkLF589*mDm46>LA~40G#ImhNw7h@YXZRw6#Gq zJQsmY!2r+Q0f5g!>shE{d>^%T!^t*uyIhh#ZlSrF@}S8xC~tQxgmW88AKxjKEi5)uEMyCB zJhum$GK&_F=#2X)v6DSaoZ?ynv0O27gmjj=wA-hz%Th4D>TIC#C(U)3RDIoQfG|Zy zQ~t01MS}DH^e@($;gIMo0S5!1_lOV-dALk+KVEvhUHb@#)snjW??uiFV{>JHR?KPg z-AO!ifeM?IFA3y>{)Z-Gea0Flc6Ca1UJaooeSWnF$m3{O)qyXgb7ktYU+g+1NA~9a z`uGR?y9Fd&Q~xISy-BG+1nq^k!$OHfu!>LMD^fU%i*A0Z0Q`?r-Rs)8d6U6`{$oo&N)lVlL zowJ^Y%E0QkuIn^xIJyhFcmBVJt@B@OBBNpPihh%7(9G4J%Ho-#jSDpRv9^FC>mt(A z&JXh{XbVTcN!*PaJ5k^(!ah=#M>V0_v#YO~tvp%nQ5s}SbRXZrXpVrE^F$duasHkC zt6(@C`MyffrQ?j2g!olndakVl${`Rs9IYtH@0bOI1yz}J8hHMpC%*;roh3(q1TvyN zB@Cv)*=&bQ5&&%Uf_D8(moparmi#Y}PjUUM$2kVCUpwVBHfjaFhW;coRSZSN*?Q(1 z*ZZGxI|UKqD@2weDy_lS4xmM}cw-$6g13^gF_dyrlDvNM<^@ys9}9L=FV#&J((}kS z&nq;DWsNpc0b?OaFAZ@aYP_6)hE%-6_5PFh!C$xbQtP0T{Dt+b{QDeVwd@KJ4{q_9 ztG^W@rKIMM1=g6~Bf+@avsH4$y9fao2UCTSwLy(`Xif+_gez?xl}5nbgEled@~0F9 zhP(=F6-Hm`{0;WjSYm}X&`}`hTR+K18W zbOfZfoRrogFqfSS>tEYxrnECMubO(|ddZaXW`^~~Qh6IQu!E>EKwDMFNOVb$*Rf&^ zAHN2Q=L)&Cs2q8u2d>16w0}A!o#$57R`%$<(mEE+aIKuLN;7_iP_o>piwt=SH6p}6 zGhOJShH0c}|4na@?5_<%lB68;5L}G9B&4!6|$z3W3J>HCUTCe{mt)(8>ife%*; zyyCrSLJpKHyRsx=pVk!mJ4|uIjpgLI=%F&zGP4RI@&QGZaM+ZB`lzy@?(L7C3BN^0 z;Q60b6E?BjvWxWS2;nWhJ+i{(WxARc;NB)+tOBi)jQA7xX4E{~g;-s-a^7fa4e{DJg}aj4!_I zf_-QObdAg$9HY81;REE35>6{(H0GT#Kq?wLPHsyV01p+p?J^N-$c)EUB#KM>8#LSR zACjp?s^!82M_<4F>ai77OpI|gR^Y5OCZ~3v)xQt#&CGTt%vGPPopoClR7kwQ6DT1n zn?gjuu-3jPLbcp08briZfWG}DGWf9ytM0N9r{VKr@}iR7s$s$R?TS!@j#0u|_->%$zsO%*&dMH4KWS@>=xkL03A3iG7-6Ol~Av4aH z(8yMB&2QfSb_!r*RFDU^x|9WjJ1m|*0sr!gIdddLa6g$ikgoclJKPRsazA{wo{%t5 zJDwVcI4ROd7&QYLQlQ^VPpSFm=``e8wLV>;Az#M)K97NsR+V}Zph;$<#16$ovvsUV*f1<4I z;W|mN@UJQ~xQX4>a0+eibKb!Im@{#a_9!@7Bn*xp3T>TZ${_xV)CiyIL zC~mA>Jp|Fy=9){%>CSa+k`ma_M)DnK$=EI{rSi?X@zRzt@pui>*2S=FFbywoKGM}n zQMQyT24Gu(-x`$Y7Qv=Xhz5)Li-}1|F#*lU^}TpPdOTP@o5NkdLXZE1AS}tZx1|~< zz74lu3gd>?iur?HEe0Ej8yu%h8pdH{^`+)K&X%$(K2$PCiEA~)ksTXrYa={x6#ZIG zS|UGmb#-Op=2mw-cYS*lqZyM@(P@HzcuX#mUfk*G%;Zp}!EDL}owFOS`hc&aAsu^0 zSRSx6)N1nKm0EVlB%ab}w;UspusR8oHLH-(jdv?JTy`HZLo+H=5oIzXffaR^KqqSpI0Yfa{I-T*i^RS5Kz{g zy?sW%%VO|-GglT%*2EkwfN37q``by`dB2mM{&&05{NADTuEsU&?fjoQbIYYhTa9WS z3pNDETk|ji)@GD1L&|x94-yK`Fuvel=TK_@%c<0V2ywuDoDWHsjNDrQYE?1`E%-pL z*dY8zRf<#3bGYF4wueb0dAgtlG}N|6r`m{3CK5O12Swq=4e;FAY`v*5=`1KvXPQ#x z?qp)k@hW&S7D=d29BKb}Ww7WmWGbBiMuzBJFmemBeH4N+vRQdbXQ601BA^8o!3mjk zg1^3OQ#|RsS)rsG0@$z$Q~q@Mh$c*0;76mp_&jIx^fwT`2_DTT&1Ls*epvXVc8KXy zyuT>*>U0UGb4s`>=^#-+%`R>l4ZEjbq{H7}qTGR^+KB?A1P26o%HK?lmo|}h>zt1 zwCo%lmqx+N)%~yOK#5-=5w5N{paQ-gEK)jUhaI1EHyh0L#XsFOLDONm`yMyd6t@J^ zX0H~Ae5ARTcdr`U=Jbe=fgbzpi|uAZrxZq3m095%Qd2Ad;86t_BpKV44sSn-(B%^2 zVm_-ADhb$B{+r+qpOY780R#W0W6e6a@G<_LF&@p=J#2p|^cgUmmAYu(ZwNmRq*^(< z1zPi3oNtrrotHnFK1QV)TaK@lKeirPbfGZRoIN11j!-v^*@|t$rgX&b05W{t*h@_UPxnrv(``Rt z6J1;zO09{%{;2cfywKRt)@NJYXiFiqwg0bt7RrCO+b}6k{Y|CmgHk#oM}09PdSBn) zLWQ{0a%=TuRrQss0JjUBcfKfO(p`?(TWr}YR^x7Omn|vIgGz6I56ezQDedwqPmzBdm*TAGOTcn*^wx*U1f!~)D_reJV7&AKp$Fq4 zVj{~_)J9^4{o13xmMuYcLSNDY$rkE~?L=!$flcrQv_l8`ANdgn4lLb=GFGqAg0w1) zdnx=zM5saQx6H4=NvFAd(&H^q>*;#kkA3wIG5kGhX)lepfW{~#au~1AID>w7gDjT< z)SMt^`f%xcErP+cLpi;E-b?7_^_jx(cK??d$8WfB#dhWCHDcE_1y^p{jvl;{_Dms{ zg02sXQqJ3Mghsc~-At2**y|h#T;xN;aTGbSp+DY-?L3LeJbd9~laRn?HCjn9wCyF)Wi zCq0YSsb>)3u>NDhznuy(nR3w0G)R(U(jd;zup{d@;y7X#3N>isTfw@!-eD+6;fLeD zt!H<}VRd`;`0ryZWV2qkh;&j?mI;huEn{paOIalk{m=FRceI7Cs880rq5TNB?QE;O zsyBk-)9b%4ybWu6&g2O! zn{N3Kd>iO{?kb$|KJmh^U~b@fbnJJ*vG#w}LGM4$h5m``av$J9aLnIA zmetbzk~dyaTaO?Z4iPgn3#0{Ue!evRi)M-}!(PcDiG<~nWP>b;zQl7ET4R3{nHMX7 z2jXSj$=0+f%Hu&Hjj$B|aS-#sjXC<%(ESK2Cg!rh$oaeO6j}2(3XD_8IR;OTc!w z<#kE%W9A_qn0xgwxnpAF)F0Z?QRkn{9cVG?cs{(w_P@7EZ`@*N$#9v@zv)2p49eR& zmpWvK7oEjws<=x2^`xSfc*(luugV#9h&a2CvR37z{Hhz%~- zA>aU<>~4ZC^j5h1`L~&i6^Gz6LWd#%IU0QAC3<=nw~F9q{!)PC*#5DDJ12K|wXF5u&`5h;Qj| z0$Q|_*p%1t`>{5AxEl9gT=F;1T3S&bHjPJGzcq(Ql2HEab2kA&__N6l(|L&J6y2>8^hui7*q0;bkQT@{0L@60q!N?*rx)^;g6q;y>YyXk(YGS46XCdocP``z}ou9jw+f6<}xn4JB&hRlviH!VM=JG z7jK^L+be4|6}jZf)%%9wRQxEv7a`ydh3DrA7c?9a3VezJRMLOQuKevVej3Gxd)fY~ z>$^0-sRPV9|6+PW(Mm0RD|Kt;PlEyLgvw_>j^x!)Qck_R*N`JU8DX7jJ?8e)uSTxfrKAnn|9cQpxcd% zdpIwU15-G&C~&-)rBL;A?aJTB*oqozL^LR`ucxyvmN7n3{bPvoJXu}Uzh&jTp3kd# z(>W{hLf52BK}uKjlk`wq@tp9u*AdRdWPr?tyaK9~bE$%aT5o%5Clb-Q`^IJfy2q;FvDyFAn0JEWIIB;m%K1DPUD(dU zf++}YUCd9DEI&p{y=ky0GWWCpMtj~v06cn2D2gF@!*)kKaRZd%CTm0b*4fM|-lGg% zb&V=$5L6t>BE0k~fY>{55QQ>YQ~GY3UR8oQA?liK|GZ?$Xf+TPxF-HZ`@?tnWz-K+YsGCk=^I+n?69uZ+Q zV`ijle%719p?ko+G+$-7AV+w^EAemzl)vhH`TErkI*%T2O^$$g(|T7YZr!XoMNwtq zFhTL#{ms$&uuE5mhu(LHMPJ}NR*jz|#mS!?_Bi?C@F@fD>y{et`eN+j!d2>BPn_LG z7ZYzHP+bp>bzj!eZ2w#H&V@5gVTz`)t{{!Q4v_=1FfK36l3EvYRbTBGF)bBVIZ)V_ zx1@!VW#Aci+hc6*kPyOp^q@7l$2K%i=gK|W%YT=z`9m&?HVL?=m;YdXELjLWtV`#& zV)mtv?{yp<;~=;+|A!YMKnhx~`!$1cC*?$Hy{pXVA;vr=evI`lKgX*-U1Ytu2RAND z7C1Od6O@$p9dAHKiu^CPvm}#CuBf!IAgFb+=DKn+xGq77Pi|TgUTEZ@8u28 zFRTFq@X@1!yQXcvYOziiE5{I8*y7E|<#o~%*YFsM!64)r6f>Zd)_o&C%DPxy-7|EtL$?Q&)|DE*iA}gDw5+vZOpSt!%J&$n1)B|aKWPT-@rfBlo_ z>a7}%BcLEnBT^HRx<(7>fNgy3W34s?nz~dov_qvr)X;}(q3apeIr_>WctKvV9|nR{<1GR}kg(Y(g>W*qx$WAYNIEv6-Z}Zy+ ztKx^g-=s`NHA8z3TG(e`FTJ=Uoi80X>d$>^eu-8n8G?!|#m;G>iy|fl?Js)KPbp_I zrey9mj4>l;U51!4i=k#!DJB#5ZZEIAA7oZ9xGfc9Z_2Qw8 zUi#O!O);tDmmo=l8wo-cqsllGn<3dya3TKf+1v`BcHG(!)eEncYL@{2Sr5A`ltimw zgEcgXf)i2iS|kWVwImzp2}{DS4-EoK2+-8vZnc(t$Vw$R`e&P|M@J1Gp7@o}jhmU3 zv8G^VHzGnxhZwv6Hv|A8VGmoGcGU?}g$hHB{|tT(B_Ra(Id>xqzC63azPx+c6SuUg z>lgCizSpyLQ>izNsoU+=KCC^8JSvY_{?};4au%%z1%;1TYprW6fEeDRr`psL z>Q2s-f+ATjIVN%Q)4z;qrWYOUc3MyzW#c{hWbB;`;U7?-myUN&q`Xy-!SQ#IkVMHS z$B@gdA@7s87|$t$xfrKLUX%=R8gvkX!og26V4^Aybx0e};)V#@MPHsELXl#wXUS|7 zmp!79B)?Z3xOKqKN1v_f-zTP#s?D|{ks!D>EDG@vq*2iZ8$){!s}!7M)KkpIjh!1a z-J`3`@S%mn=C0CT`p0On-Ooq(@boVG9L|5OZ+=3Po{|ez&eIUC+zePWb%)vmNTZNj z&;!IoyehEmayooGu4bq!85kkx0nr^gMR}H;`Yh&V9X)XvL@dLn0XL#%2v9wPl2EWp zeuKqGVmj?hw>5=-pA><)oYkEpF_7L;n=@t&&2G*gHm7DR4>ezJWwdl&-l?k{Avcg1 zUdWe;Ox|yI{3o&xJYT=I4134+>fCnKY-dVhXg=}`X@!EdsRTw&AoRy6KvO!ja6bjo zD}(+L#Wl2b%G+mEm6CS;vJ_liO!SM3fZ+9`g}~G&>li{+3`&$seuNE=FA?0yn#oM zO=eb}6_F;#t#eR}=1eC-zpxjl%&yl4k5tgQ5t-)t`)%*Wx)do9t4-k@qGzvbOde=2 zp^$*&#S69f!Q4POR16Q>9A=ak60yV2!qGuuO{OW-lq3G8cMko7V>;0*MY=o~hYy1< zO{k~sz7V0v`XSa7E)dl_^K(ZV_FAkTzTsiwPY919V|uC>QSBuy`O7DOq|pE6ngLxp z;K#xX4g9K?Np^U zk$%*NWI)fQgd|d*tWQVSvZ*k2G!ya5`$4o=rPLoojTMTN``6V~1B@1Mbojw?&5NlA z0NV#l2fTv8&*2D$j;(SBK9Y8U?PVwd*jeDsZ7vBv{>HvnZ4&WCU@oCk|$8 zI5eV9H^aFGis$e6@Z2TzB)1M|H&Xz?a7}9J_tRcySJFYB^>DHxfD`n&I$%ESe(FM` zZz6q9M~w_%ajlO9tdoSwdY;C9g*1$LGk}N#DUk@Yc=^Nkqln!8SjkKVZC-6jtPOU! zsK%QLT?s=}Xi>jEJ{_(~Em} z;Po{LE=FpHRaz4Cb&sB)lIwat;X#R&`lBWp(|3l8ZpA;P30-4@LOi`KxW7+%!WG0H zx-tZtx2Y$7i>%gFqf83tC#4UHnP8F=fO3ArEy7O2&55qYz#p$JCs$jg{zBCtPuex9DI=uW_fn(vDf{)NZZSm&KGVO*AF z=={kQlPm=a|N2Bwr~G#7zE`ZasahPeUeEra^8EB$j}GA3Fn&60gOdus1~9-}#2y_V zES6vX6Y7myfPS)j5>Tbus;UJRwZ%GfgD<0FynJ01(%ee2{>? z4gl_qcW;tL1JcnoJkBk)6mX!0c4W49Zel!uU;Lmkjmn+WX771~r!+AOAy=Ta{fP1G z_5zNmj`Ao97U>@M5dow8@N=9aYqoo$ggCGNo%^%U)dhFKfcbp(o%%@8nOyk6PN~qB;|JeB}kAx1Hm~Rp@Fv zYgOAnrfk=Ylse819e2A=GP+$NkuO`H5Job973QdnSC-J`PWoA>GxJ3f?=9Zkyzdk4 za4Xh~&uf-7E`ALmr~Q`uwN_PVD$4T&plO6U6_0-u8js`l7R=h?1|$}n0hG^=7KD-*1^*tWWazVi(U z?06XwtS54{MUG(_2QMI zkL=@Jr86i|)Cz}8T>%B6(oLoLzG>fB6eWAEm&VTEJ zEm3HHH$G0043rU0Nd!+14cQINq)mMk9h+^`1xBe@z&-ISp8w!!ZzTeosx}tY2XcNT z!19%cgC1)N#yaJ~sUx7_k$=m-u##}GgY19yi)`Z8+4siHd+AvtFr-qk!=y@|05N1y z;L|L~AK3)vYz3gTA6M+zgnWUTm2ipdh$Cf`)e}U7%f-}L1 zrw5QbvY#GVD7d^>LLeqpZrF1vo;!Yy>n}Fvg9vZen|yAmBtUKhC=Y;)8oiTNR)&y1 zLj}q02r9>TAbEt#_Lg}9kN~${PAfN&-%HHa@T!0;`e!~JrvGafzY9XJ>c7vz>)e~& z6)wb>)iw@IjTZh$WehK3g*>cln5H5soKN4@^5Xisdh!2;R=go`K|K}k4Re0j?-LZ! z;N}4PqsX_cNOQmNdi-f+JJo@LnIqW8@1;!y$|rWj9u@SZ;E!CC60q? z5PG5vfr=z0r@Ws9O8m<&Kc`w_?=c-z)kj9bA*$#Y(5xd0Y7Mz>Zb7?$V;#hLA=!vH zSS<~wf-ehV&NK}h;@@W;q_HyULLvp4urRw`I5M zpZ{!oHBG&V< z;d&|KuG9e(E8n9K;39`W$sSrAh60v(R(yWJR*d? zVZ7@)`5QInVL=P-cd)GHKH}q)%{fpo4FyGCCLLzbTW#igpG-z6JOOzPXG0y;599)m z{Yx|UpSARm5U=y{a+W-(ryqjAQutUwdm3cq-bzDmd$LW#ocxfch ztaiv*EMw-1*$r^7-=1vdwyO;z0y>2(OwiWi1BmWF8AX&{^P0l4{H+1BTQ zVJCTSx#!NSSX7Y4E)8+&CZ?E`WFOvji)Qotx} z_sj;LHkIcu0x8_w5LhqXe+ws-Q%BvP5p1MO5`Uh=Y>9(Zi&ZZ^`g%^+)E4(Iule8r zQ!HqBSN&0L0R{>c>4+I!68X*rDqzb-4}vPtz~-A`Y>-8&#dlv*EH&xmL)|>gL+6D$ zxWa`i%yZr7=W||VOL&QF^5)atPKwa<+ivly(B$fU4VhUku6D=zlkO&sN!qiP#`!eD z$Nui;#x80jy|ekd_uATwh+(nbn44eMN$qJjB(tUS2rjJ&F0-h`34u87b$`UiOw6y} zN?yu~(qbSuN4+8);L)1=+1?pV!d&zWHUy+R*dLcwOmak~*2wTVfAlz}Nb5Cd6;Dp4|Z~#WP57A>AS`KNX)OuhW%I zO(P>t(|}t`6+0RjK~gIdFb>W5G^bgQi&Tu`{6Bd^Hh?A#UQSJSqR_hA1vvmIFLw_~ z9%SL46QcWTLkCUuhq$l=`z4V^3H$=t}BWnQyFA;$(Hv!C=>sPO(lNmB!7ex0R#}(j6Mze(78h3`wBAK~k zUUv#|4|-yLEdk#vD5HGkX5dczgAp%WdG2*dd}#vu&U3k2e<;@0JY_t6&la_&5r zCv`knz0;iM#Jq%(vMiP=!yjtt6u=@bcd})?z2la+Hq_^)@TQ?7D{RC)IvZuf zELd?ZHrF&z^w36_MlH$RE+GMYgO|m$X>g{-o zkCo}{`S|>HJ-0lGcYDQnUTD$3>q++iX90ZE;%?PPR&WTfVz%3AVnE>{a%u^U$LEHI z?pNy>^5_g%0Z-Q+9`mHx(|!ok@QWB2H3z-4plSh*<2Y^($N` z2;~(8X#MoOdk!JUeo33%e5VU|DQPl^hbv+2O~4hD8~E7DbC9o|J1Z;f4;%|jzkxdn zk08TWJNp0)^8z5VFZa?HO724%K@h9HNG&r>I%@!y|K~(en%f{b%>^uVMc>2(g55jL z$+m!Nrphc4*aqhC50QfDqG9h5`Y_>$S7h`kn8qnu z%_C)Gk4RmBk*D!2tH?|w!_rF^LCoNdk(4el&$D^HjN;az3hN;m2+F6~C4@@0pv?G9 zk^Ys@W@d!weM8KPB)gW(9?p&tkt~k?oy>|#zK>8ENZJ!e4Wbv0Mjb&xm;M01sKu-( z7b|0p@b^IX-eXz`9s%w4bO;vFp-(Y>J$9z9yc#OxZ6w|b#$=K*edkT#nsp$Zlb^5t zZ;BJ2^wM4?rRwDl``b)~Xpb#vO%p1CiOhhNJ#VhJB#Q~+h?P0&F@Gh{@AZ%Das;<*msNh1@`_%eP)|J0@SzD1HaOEjF z5oAbT4R3GUtxTzM&7i|{U?4JtbTvr2QZ%|ck{s6SuKSggYnuOUozZuLD?v?!0BT<2 zqQe%wS)C=Pu3P!^ldo$3$_%wzwYp!uO1}QGHWXDUYj7Q%`7>WB9T=A=B7dZnvdsHQ z0)Mk7{MF)NoUpS9_zv{vg?0@Q8HBcgQo8RHvD5cPZG>y=sTA%wx)-M<5Ws5P>junj zPnB%9Pro-O?3`S%TnJ@Kf>GIW0BELEJg<-e?25`IJP3W24T@wgjo<1mesmIMA3Cu|A zr34Ct7&L?=Gg%e(XCcEBN(D^=3eA^A%`TC%|JWN5u5@gR6cJ1K?iD4imlu)K^!x$w zob4!@&rLB-u*qmpsHfAm{9-`9eo=3`DbH1*_clIGF5jhrjr%TC&1<-7K%xSlxk!U( z?;8!G&Y63G-yE9A3mfEi_8tWLAi2fQwYAr1g%DRaXfY=lkxJ-K;TH}8USIB~-u(lC z$+7em*;@mDh^ge@Kky2hAG7-<4fh;;z?uBP{81;cJ)0#tOqJ|Kh&pxtC^q3|52|S~}*7MLg>pm`=W+>WVS; zYERrEBGSYq^u&LCxUdlgQd!PV+D{KwE2gUHjg@-hJ~@)zer&C=n(sON>`t+ntFWBU z_-P=Dpx(y=9baP2AO!5RzFvErOQDGtKs{-(j{H1WeHJlM&!JFCxEGd;tBZoyt|^cO z(MIbzmaU=Hm{%M+GKpY{!%hB~=4zM&DuiY58~O3MFM$8NjQ_DWy6Cvr^(XtH|88f% zR%8fRcfh2d-4;_6n61VU*%|CHTg|xkz zm+_AH3Yg%psWAGkbd5byc>HzRJx9C~-2wB8>x`=42y{Ika?YR?cc6t3Uy9#_uHG!t z;60$U`A2@jJvpuIhd67e$4O+^-uJV9H+&O2D!Dvd`y4lN$uDm{Z3zRpC-UzCdO&)5 zG}W%+3m76jXVd(OlXAVtvTyNKcyL|Dym5WzsCz;jgT#6bTIu~$wk?|)-y;qQH``vY z*#4_eA>JaBq|IudB%*1+&Ya=Gvqv&VKndla4xIuY#P8;~e-crT_$a`ra#OGtd4lzQ z1l1pSb)(>G@A$qW5e~0Qt4O3eDM$a*7S1oBWPC!kf_CSmpaQEF@>Vl*xoVB8OG~zx=if+;z58 zWlk(j_4R!I(uu;Kh^82N;|ORBenBQE)_dY)$(PTfD^VBm@2_3{NZAU??tkE{lu30>&fSB%+xPXhl~XMe+lZmI`fZv$9~ zVECaBmJrsg8L$D&nk8G{@?xI@ehorIQc`4ZKsjqM|6Tta=pKGEn(DEOYsEC&;&P8F zW#WIO9mBbvmSoX*d$C*DVb&i((#gY5}ov|V2m7XvY#!k z8E;o8YyV(khf{1R0fe6B96j!Xf;)Q{<7SB>p(k(sN#O9Sm)LNmB$|#_8#0LWJ*|5Y zYe1i{vw*q47wK^N8R7t7bZFZm-g-R;02F9ECI$utC=>b?hP}uoO`76tNcB?6X%fr? zdbP1WG`kXiH0UCz=QMp9S#@Y9LUXc-#}M`sRM8+k_*FR44-LIzd8CQ`tcM6N{A7CM zz9@WB72SJ+1dXXl3ZF{c#w=^{lAVeQi%XQN-gZQ~!4(}DtRQ^%_ib$UD&-^rQ~ris z7V66&>gfm`d>r6Op$*iQi_bh?|u)+ApQ2rN}IyYh~OcDlDZ!=-yJv z(9FM~#|n{5AxVkqDo;LP#RmxwZm!`rf^sp%^(5U*Q;q8Pse~;bzXA5aUc8b{c4X(> z%mzu3|Mmy#&%pj*TKCK-hH@VYOb3ITR8#606BhmigFkmC_X-7rRera9oMX40E_$P) zv;aV?Qfp|12kJnwyct(8@XWFS9T>#VFcj}b8UxptLaTLj=EARex&PWj#R3Ks>aB)M zs&37C<2H6;8g~?APyjEZaV5nIO_DQ7Q;fryt(}r)*wx9u_9nHr71M#t=NEv+>T_fD z<}v3QXFU2dDnXzVldzl2_HGMsn#|kaJl3ORkl=VLHse_Sb*^4bt>Peh>#hx80_Q4){xrRrx%ja>uiEXUfnW9 z*s_~uLdeOJ0AZ2KeV~K|Q_+#QDWNPnW0y66;X<rK$F!jPFw-(qT^WXlJzQSkdpBfdfSY6mFrv z9kcHyX^jXH36sT{w>8t#the4eHF4Ntlf>kUyzitHj^jo7(i5KmPKg4U-PO^8B$wKN zY88Jm2wt+;0M+w?_5vk+aHl5@9jSGuvFRBiUB!+z_!TxB*D6yo!pqgh^t0J;^k$g2 zs1s<|gGvQj#%;d7($*w)mL_xz9=7|-@*`o4$|nb~H0EvF1%r)a(|UR+c$GPUsBA$0 zt0*O9uU=J(41d2=UX}l1_Pf+7*}fS2j(81FqQ|0^^#Tk)V3&3n0i3PcTgx6+z=~V& z!x-6;MIlJ#U`EPOtU5PZdTU76chub)8L}ezJI*sEW20sL^<&`rC#HEcB{@UDcz?&7 zsQu(_>TB_<^7%zHNb@k|&k&&n1bsBGj3|wH#BG<$|D&V(FUX`ms;#a}Z&|0YsCE%; zJs-S`|8RPvA825|w=X-O($5?ZF&P;LzAGKHzk&ug1n}o+r}KSXdsEKhJrLs>H;;X;L_45wJh9DZc5nS+sWj8=tc#=U?Wu zM;V_QL`Jp!Ves+|(l@)XFGJtntC2{up#_zFzB~pJPLd+Q0eS&X_p6bpn}2*g9(06{ zM}GK0H(}#d+uV1{(w{n|XcUHE(MM7}=aUx{bGA=-qD=DN9^A73Kc?O?EUIt; z*IqLW3|&%EGlYa9-2yX6mvpCeh_rMKDGeebEhQl#p@1|Bh;#{xFiMMb_qW)4pYt90 z>2>K3)~tEoC+_FNKLP*1zJPo#FeE;^jbdd`?GV}9o!#gpyVeno1*Nqe7xkURua8&N zUUkptaF=|hd!2Rk7v;s?v_-vO3&v`ctvda${DqlhcDiD6Pf~DnOM1saL6j~iRr#%r zjkc+iAy|e0jl>tn3!z{m9!!@$f{a_3FSCNrpcTmg81*^{>rt}YJm^ZcsLJZ^c4A}T zIgxcw=XMV~`e(dI&#@7SyfYE{mi~h4{{5GB-!h!q3T!u}jXehzf|QPAg!^b$Z{k7K zi#Jc%%734-f}0vvl0Z6fR|FpdIw7N_|g5WDIv1vd%Msn^< znfl9H+vyr}PFItU56gI2DS>Bt26~Z>TG-dZvaJn{4;Xjni#-V5`6)xtE%o@0CI-GCCSQ?gAjHO#inZT!wykKZ0ry63-C80J9*RPZU;f0 z-~&6_y}gnvzh2lZX_MMw6&O-7Bs1Gt4AsJOMcry9hr&3`B376(1Jr#BS|2V3KwfGM zg|)3^*j|*>FMWBy_rh$=>C0b=i`O62zL-!j1C${hi~eERZ^&gBZ?KTR5NCb`)7RhhAP zcNR@Q80E6${%4pKQk6aMZ6fLAheUe&C3p?J?iZi+w5-6~P@$mHa$+B z+)@uma;t`&H$Iaxsn{Ccj`@&;P3HVb)u7c3E~2vKhW3vCW>y5f!7jY<7pUK4AWw<) zB)ma8j>`Q`rk<>^AJABKS8%=E=W z@k>YV64LFM`*Z@G_`@&{`POJiG+MK2tpU#FMa*EMM<7^!()uVuVfRVfk=^TLUkY30{PL% z7OzB26H`SxBFjAJ!%?TXJ0HI-xOM2!)2>9sJhlc$?>^Kx`@wVJr1F=J4ASu1uER2x zpeLUGu;ZLSx5RZp?jycrhkK4k@9n*5Q@1EfOw%EFFWvPkO!B#9=0&6IOC&%4=|Vi% z=F9`k;vEG1$i?;@*zMb{4{JEaKb5oG~d?nGn zZT89{pu8145shtUWBbyw6`noicVRrCXxluJn^4{iFJc!Q4N-te3o* z&8D&OD`2wJq(_K%a6Cb!qANKrStdg|w0-2G>M+G+@mIT(Am1V5PVfh*dUGyQmAjMn z?@v5m#tkNgD9_S+Orbm7=sXT~`c*T8^~S=^3HV!HF`g$KcvH`%f8K?O{OUM)yo$PR zKG^&z#`4D+7rpk;fU0u>#NYcno6RwEg`)=FEz*Zj=PpZRr+yX->yyFBihHd7`bfi7Wv)#4}QzaW+mFb^uUJHpmsM}^|_MGxrOWU+ESwx8i~ zE{iBG-0&GjR~Y>8H=UZh76Ze}&RbIvSlqE#p6UtNo|w0B?flLdg{kISG&*7c&d4a6 zT#=Bp54HpXj^)a%*4@U%=o!t%5)0q~K)0T;_y`}`D3$VHoSZO-yzaZXOQ0!fYHRowi=|^ek1#a-`Fqi4UvwqOdIxOFET&qk)MB@_=L#6!CAjOX6^TR)Z_(X*!iY0~4n7{}I#dU%eXsei@GRZnbzeYj>lRoomv%^SstHdotIg z4Q}_O^MBKS$p&4DB5%=@y;u~en-v&|L1Y=Y1+Gm>HfzLj?&vst^2aQk4L#RnEPLNC z0Ra*@!rYfDvjol{e3EA(X5EXCUP<@JAy+)skAR{I#u~veG5~!yWu*eD8AncR74D_c zRs1M))Oub~wk$x)Aa{RGoO(4L8JlM__%oth4C4VH--SNrhY|6o#0Ee4JYh~7(8>Y{ zZK7Zee?pIIYQGc{?F|3~7{nhY;DoLg2f}OAg-GEigmo%EWdZWd08}j1*X~=Nof)FR z0%2r%_sO^EnNbR`wfyDuwq=38jtoQ0pHjhyA`%cSx z`{#iH-@z_RkT2tp7Nz-5>Pja^$4P1rc5F2Dn+H#<+CkFYjtwbc9+`7iPRw1=ms{!O z%7J#EPcVCtZLzqFd8EYrLre?jNAQx!R_1~_QJ#Y>c)Lta?V#ym9&i4Fb1?(L35XFg zX5ej)Vc)=|q(JhFFtzZrBv<7k?50iF_}!wC`+3p4-xucMwufC_4B&sbIO8Ip#leZ; zb(R)NubtYwPRU*N!|>pk_LWBMb>Z633!ijlp+RMiyIB%;?~~6)DOqxPp@n!g)8u=vvELHNgr_l^$ zxlrLxSa~F6FPWH=UE#qx!#I`3h$*#sj%Aq+astt`_k5)1>tY{ajiLfrVWd*k(PAS! zZO;zmnJ!_RcT))!T%kGgMW(>h+Gs@)3;ii9oIi4r__I|Tc?INMk9;&0Z~_2niLoh+ zic#b|;lXqy-2#b(EXYC=ih|!nQqM)#%9z2J@=pZJQ9k;$;CbI8@o)5ZueLgA3?VJL z9zUP~d-%|&FLY+T-5L;QcuVNM*wqG2B`AYW(J|%7uJvb9S&FJ}-4J11^qFugD!|9< z+|TBX>{S;EOg{x{G0yEQ7_wDO>%M9Ixv)O}C@-5TkUl7)|-j0W)r1-s--; z5te%(qRJ&V(Y6Fwc-6ZQa$Yq`1502qQ4`x?^W8x^1$~Vo85(?wx=~_JiJ6N&LMX)Q!>fV{h3c1GG7XsrY#vwjJAk z=<&2}PwWzeh<|#lD_OPPB|7Y#v^&X24M&m*`()kY6Cd#?mQqAzdL;Qr5TfYS0*mJgHx5C|D zs#Q-?qY~V(9ja0!U%c|@E)$>rc*|^7m=rvoLX7jlW1kr@pg&Jn8L58d$d&|PF?$37 zuvD+gxD9bqtq5=Z(d;pDlMR@{cs(J0v=xa(skl@VPQq8&0G?^muXu?`a_rAGFUbJ` z*2?Dw*nA9ai`c=Nq=6FoH{aKy`^@)_<=*rCcTc!pO2t?xwsE(Gef|JrI+#;Nd99`{G3^ z?8Y3(CCUQ7NkP$n_@t2)6%e8X4d(d5o{~C$i$k5921sr!fq@Wk(bUcS(+Ybg|JTar zGbkmC&`Y1@-9LYO?bGxZ~a7h<0wXKJZXgkau$% zlqPwVVCRi@!*%flgQ%(@A?+6jZ&5F0Bk;b0|e}tKBwlQ9Ta0`w|{v z`?dXLP0};1?jeL|lD598ai_%7$ev|4NB*}H;QCl7d&aBuyH@X4Ix0)Nem+X;`9bad z&rsiy3WrKK2+44Q5iC<=$Mav|hvG;pK6mHK`a=L=UbZu;#D}j)x81JjnlvnplXwhx z;`qri`B=^bi;RoEr+c1MS0+|~-|t|f2e)K*5K85XWWye;R`>U+#Vb58K!;k;Ldjqf z(B^`#;lepW>3OnK8oSz_C|0nWY*l{ki2n zjh2ltG3|@$Rf37)O%_T>H|pTjzrm^tI#ReRec`%HD|q_aH@MV+dEO!~$IlT=q-6W% zCe)!h!V$W!hF@y>hL#-NIT3JM^B=EK`B^-Z4JxD14+j7A{V(R)-i&YfGlDZwq zKL6ib&s(jXRt<~A&qOFA0cKz zRw=(Y8ryeIABNs`hLn2PA1GCr7#qJnQ_{JU?|B$le4H!zGJyTzv)S(NCARarGA{6t zlTvpk2Z%#B@dxmwCW!s>ANApI%8s#xa9NJ%12(5R1SeJ_CYE-;nJFdAw5a>39G;MyCUGqZ?0Cflec z>Ap<5RrRsU_)fUIAC{!{VT}JvFv#4AaJF=ty*osdrOI3p39op!n1|c-9xs(D4?hg2 z>LLDWJh6?>9WwgoR(XQke{V`^oiDDexirQVMEGh>0pGYlX19hJ3p)lStRn4hMg% z!7u^uu{VAahfXB!Buj1`6>+#eOAZAG%;rgC(%&923~H{}mn>y^Qu4RGII$n0sU6$t z*H;(D7FQkl*^pKToARfS?Ugl654n{RA($LLU|LCfH?20#SQ8uZ>`!XoipU2FWDF%B z==wt#_i@iVx{uYMNp7#&$Muy4Ou}RLPui}1gG>}dA|nw03t#BML$H#+M$g&7ch+f$ zMsbsJ*tZ*f{KIdi1e}wui$mQ~^|9D(Bav$TP`2M#CGO5-)KtPn8JN}q4VJIL$GDX;L(A= zCYWZ9QC&m~Dj$4c3cYzYO~r;*k=m?)TBd!%Ip;~i_t=)O8vTf@dI~X?Q5>I!*CKe3)9oL8C85bZdWMfY z?{{&0lk2*b=W*-Fe4}o*RZgp;%hgWrm{+nvO-<>&q9hg|QaDBkT3l19`8dI(!xtVh zIr8+W{@?rL{@W|R<(y}a`&73LG<0grGV%WZj1GRtAVShSK;-Ey#g|Q6S#1lBU0$d? zoWiFYffYHwLePIAC5KfS(Muw{PbXEG;hih2C`#Ykf52YX+4@^Sn_go7)m&d|g9f~l z)R*^+l@=lC;PMi1VT8dfE>!P(e_6;@y3{M)?9B9Wh&=x{TOYL^RrC}z(tqeB)jl>3 z%fF>Ap|7Wxo|d?~|D4Q06r{`E*{jl$NmoVMuZ_;=ZW=SM2dGP&>~87IdR_Ol`+G8- z{QlVuW|F~F^8C%SYXB6t!nPI=j_e_j|Im zh}q#cA5ZP#v-_)T3^9X5N!}Qm0H}imfpKe2I1OWT!d*o>q)x=4#9i(EqLxCFXaD9{ zqHwvAvCG&WJ6;T=fOoEm5!ehyCQBEQ5f>c zEP$#r_GWzcIG~hpP4;^S2AQsq_=YVtf>ALpjFb(}D14q!)Ng}CD@^6xBH6ONO(L#Y zg8_l}s6^|2&RGg+d_rEz3lfm}0*6o;;q9^j;e>lBq(o>EHhKR1C^k05hUnJnm@Amc zPEjgzbon0AH&IkO1#crZI;@&XTl92^Kze87oeiQMQhc$eC%DhJjB7K2Q;uF$$YR6? zv`67Ymm4@1Bus!mm;hfcD@qzaS4*T`{n&^4#l9+i-9B=dBg1636Dq z`yQx|gI&~dd7`|xM%0!UAWnf7ls;QS^Zq?D2k@Eh{3U%mVZyXW7dG-ka_6{&4U$5? z(KH?ST38JmA4^!ZTua75MZlr#Vi&Z$vj`45mqk9uw=!Xue+;aF5;_JEdsWu$GUVVg zh=^y-2fE01Y<^MT`WPuelYFPBx}O~2EF0*t`p^KR7`Ox%GB4i}>F*z#w^JyU8TeA0 zBZ2>SJJkgeKfG5@gYj*BUgDqfeB2LB8DVgDJibKgE<6kzghGRZXb*Q3M&b8qDVV5V zzB9%+QjAi>V(CSh@fEe;JUaf8z4+wv0r&tBQMj$p8O|b?n>DKf_?WGKg2L1aI{xCG zHY7|_Y7*QLw#`q@QrS_W!a+D+#~d2W8;tYo&&I!k3l0q8pNkUtx$92G2NYuTIH zVPn5C2S3S(vmn0^bu*#4f7IH;VwL&iL5N@h1u~wNFN5J}ldKD*3*W;H{H%XcVd`Nw zzBy+1t$rUT>Mb^3>JtN)O7CZHfK{vCDIvs~%bfT2Z0?|G2R>rqd?N@j|CF{hz7+c$ z{QWBFQEb0Y08mFEByUB>Z@=8)?{(H=H)adHpSHjw(Oda8-HNd@`NkB{-~&(FWR9uY8!utHtSekNuF&aY+p?5P9X;GFf%&$5|yAEDZReweZ1M ziNoEh$A8ER7v1W`q^oM9ySO2$GsCH@HPm3BNa2$YUlafC0gx%^V_Ob~Kr z!hpcejIUGqUL*O*)B(V+(MxiTwGop})R6yUDY+WJhRI)c~Vt(=CmjI#y$M-y+~hDG>QQ}7#o`q-e(+q$Lg@42GD zjFnZ`A(K}}luG_KktK@dWUG+n?=`b*ODe4tYE=^dg5z3ojOQBLp7h3Liv@?nsezOu zAm9Bd2-R4lYQEXGjH0#B`uXL2GxhpO3ena3L5Bxzp$`6QlhMa|FFRzvJMrJ>nkQ@B z9^-8a&bmVK^qE+A{(xot+x=Pmke%|2Ek8-rZLr=hkMS>JpKuJo4@ZF2oa&3eh8<~O$?Eg7mtZR9A4p1CiTkV= zqL-n3m>r6*78upBPv<%=c{D@<0EpPMX=67R{)rC$+v%2n^RqE#fk$c+`2W_`R_69IW#; zBqeM~@CWpQ$}haV#my}_NcGD5hM3jH9ETQYypU@sc_jH_#)3I&D3yF^xC|s|)P849J0HORr zE=hMu2eAgotH1Z&FNy-l-z~HEO-0YhB+&o@$0SDt^)u%sgdYShqY1)xh(>NG%TBD6 zV>dmoO3A#hwdhg|!qt=EtCV=DE^f%qqE7}6D$OIFLqqYu-PG*%vo=)x6XU-`ga^6o z;mixWBSq`{Rm-4@_7;s6K9C{N)rz7;aGrEwR`16Jr#1QFgN^seVG?GS3k@+SS@IOl zg;mQvnHdl8(ymxzzy6GEK{14?0sW)W1$;vu(0Hfcb`A##j801m6_c7GJJ!cTEoETI zxb`i{Upj&is+?zaR5>B4<40?q$;GdK5O{*XkFc9SDds1ZEc7HovK{p9eo!gbm@a`# zXI6(CkLY5-y&h*pz*ZU{&#vxInh?^65l#*g3wa_wSta+AO_*v${x|G!kl}4(z`8)m zqfj5uiSR^7t8r}@cu(b}<9;klZcWv7m+Hhm9bR}}eb;lX(?d~}JhBX6W2EK>6ubYd z>ZMB=oflCm4g4b7DS*|~5z6cE`%pgzylczECgj6^{JUjwfl-W#z2F*+O*RW-juPkV ze8CArd-CV2Euq=O@n=RQG@s5}tmX8P+0|*2CDL?BuqmCuh#=&+ss+%i^83oPutdot zip%gQN!cCS<1KTYmbdb<!XUziCW+|q3p081lu#K~b)(e?LGrr~Z9%?JA-g^# zOi0*5^!Sk#%naRIMuK67Nnrqkio+ZvQ8+nQ{~B3>z$SH9l7r>B7KbJs;q6$($HfHb zTjRgOX#Fs5q;$73Az6jT32imr2|UJ#NF^R(I9l8wO8T#K!3P-8NH2$V?@Y!qnd=le z8`_}8;y$8KCo<| z;#gI?;QZ9_u>}VwQCO1KMxVMVJLC1ZdxG)N0W4@fXHz2>T z3TWt_9`khf&ci(!rDgizT~Xfw7UnN`yD`%jS6{d9tnWxqn)BO0N(27Rw|x>;TnX0` zo@Te+l$B|CHSodi8%^Q`7ccLl5_*DE^Rw=FowEy6@O+d*KFjH?UNDll1@b@?6H^pd z7k#r_{`+l8a3g+F+Rg;O&z0~hz1lqn@iNaW*|~4Oy(KjjYBiJilgbDty0$O+Ja;DC zCZ9K$Q35X3Y>dHSy7weOwKfbt9EQP>gu96)o2khJd0M)Nr{vkVKlzzO`Yu;Hik_Ab zQ>zJt(LLmcr;?R@47dI{1&FN?e*~1t$5{T|!Y6KlC7*s;3;IfvMArxw2|xiM*65R<-$Yue54h=5N>%6dzs3b=SU= zZ0jXy;J+BYqh>KEvPFUMG%8~CwCuWzkFj+nL0Nkg$K*vGK$;Ab^(GU4t0OyOJF@jl z=L0Q(Z(-ImY0*kb2-PNKb)lCX32HC!vXCLPb{2+jlC?LHw({1(I!O+~7Rdi0wLmu# zLZ0Jgs`_n{Vh1Vyd+62)R>Vm!mdXN<1`UaXF%BsQ+l0L($sN1j&j|0_YjeQ=F{4bD zv>V+O5)8m>0x0Twbx&y*V%yvAf%uT+Y?h z&QR_-v*W>i%UhrU zy?H4pg{}|PB|*qj`}>pQq?H4GT)6fkLOZnWr8|>v!c#?a#(@HV!7y2wMkDHBDTzt5 zX9-QxJc4P}V~gX3K>B%H)XS zi@e8$20lu96R~dUw+mbUxEBU;>F|vlO*o66SyOmwghx>)C~E=mjdw*BMbu4(lIw3c zkL7ow1Nr*Dedi4D>{JTB{g14;Wtvsl#H@`NAUV>s;s$Qpm2eF_Vpd+j#eJ`aCM0g| zG-gI+8$C-`3p8YTMYjX{_0fWXclDJCW3!mXxdJ&mLk5d7)Rr5L z#&fFY?_iL8_iHaPfpHG!2%U8wicCqxdCegh*)U0TkN|y**b}G-{*maT2BsGk_UMoM zgSk$6NV6;~8}I*%V_Q9FVX!H8)mYMKJjiozZ`k&I*LUBG>?Vx242~42%&cN?P=d$v z;*%oDt-j#sQWD7htTPwH#YW(wek_Zm_N5-(HBLn!w$2_7TZ)c=fSYg2dEsEUrwDQh zkAtD`L|yrkiC_QiOomx3`h_9h1$uxjw|Nen zpfmJm;W=B1Xp?@8Q4`nw3G*9+`st_54{P5Q!F%Azs5JO8UvtG*y^?MQwOI3>ZT?(I zpYd(s->J+nWuwK%Aiv_QSo%3Z7%_t=FvckGH_|O{%I0CF{>{1c5-M&EK2U7v$z#n8r)lQ`K>?r*?X|9h%Sn98NRw!PRuO z)!*xfe27tE4}?640`?vq>+#4`)r&pMYHmtArs&IdU?-gEH)F{}ZUNLuxCKWE7mdyQ z$Xyc#aIHNCaHbE@xum6S){pn&!qS0@GA@w()Lm(Q|cUp}rs zgY!c#x_YQ;$>i^*vzZk)6soI2XlkRQ?5(WRVL2CN$nF_@`A~8mv?RBDz|tE7(A+f( z^1#xDn-#wrcx+C>iucS7aBV_~Hc~@i!aw56Td9_i0A}qLl=>7q)}(FIcEmA7n|mJn z4v;P~HX@5cYU1knP~g8YuM-$qqe&$1N{zO+jn2m-l7vB?QaK{y%QXwf5R()X_m`^H zi5QkT8up zS-DO*m&BB|;MA(e8b{6ujHokepn)XtbHk6pc?BL@w?b$4$g2jNBH05OcCF|Qc>snp ze*Sq7jy75g0;xi0b@tC!cdJWWmf!M)F4ebsJH8*JU)X6kI$`k6#k09-jR?rNWmsm0 z+|Z9XI|uYZBTaJ%3Pa6D=sU9 z7lZIhIjZn@W^uNCUzZIvFFV-{@|4c)nQrP!;@47Hq==_}Yj0wfcKhTs=M?K7v2M5( z6JA&iC1$)2O}g1^t9#so_jvP58N;}_(-1}WO}9z8_&%@WZ4n|RF8V`R2g&f8jgDX@ z{81!+_r@W3?{JiSX9GqH!;vcOuRlEfb=UPrgG$|Gia3-EygHYWT#ACoN+RxQbig`Z z$Wrp{`%x1dBIFw_t~UYx;!igR(8zn<;EKrx!b@yEma)!h{3M2?N0RIy#_{fC)!lok zha(G%5*6!dU?&qwI4v@S4?QthQ$J=H?$-1v562Z(FZise#D)Ex>tYK-Ll)u=cVYW(}-%zu~SWKYff&DR_7Jq>*vHo|bXV{oUfrFIm)opOJk>^aDVOs`xRa zO1T-z7t6~+AhkcWGybaQjYaQ)srS2{h#JyhzQkJ-a?4=anjF>{3juRqVeqy+M-x9r zOJOL02Yl>gP9W`yr$7Q;0|a5fX#bERAoZ7se7@;lvzFPEUQU!xmehY-G~psl%e2t-?VBH13*aQdtsj;Y0X zKxu`Uy2!p(W0C+};Nssy;ejg6R#sm#QR5;RDj1ZjyNpa%?(-x;xXIkW6@)yw6Y4_s zH5>j4FJMPT$CVgcgp9#O^KV0#oedlL*4}@d=M2##-Nve{LRf6yakH@wV6ymztgS zuUcdlakvjyGkym{E+Cs-10AV^LH;d+188LA7+C_eT(6d`U<{d%Fv|Knmb9$bZ#}Oq z4UP_{Kp4XybeFR3VHQD2g*Abru_OQ!{{yvrL!`+=q9R=fEkOZ22_RBXK5A{g);CGl zjrnUtqaYoDwa#NZ7AQoU-lC1N)Q45e$|X{Z_a?ar1BxljEAhkJHJbq;qoN4z+9rpG zbUc*9!TQdV5fJp<*HEHWm~Q6NdtwahrJZ8%K=y$GBOEOf49y&$ptcIlznAe1`jaty zDdkr!M3kO@vRoN+tc65^7hx-1DQKrkjNT(dtAd(MjkHo2kc#>3B2zc zW$=*#dl6y8r+3pH&uC|j-7+cj9w;D`s^C2vW>8w4vN)LAMf|gOb!3yK!<91$L^ws_ zZ{^C|vbPlm_CDXz#78mwzqjS3wx8^LrZVmPmz~KPB@Y^zBfx#Slikm?>*F#VPJ)B% z+o7LS?1N645T{+qqcJf#Y<-P9kN$6SST5TqJqDh?2pSNMXW%!py6+)~z5K77yx|7k zSf~^yl5W&wP~w@e*qpV_*mZNFVXRTgr6@hI>)Hg_B{BijpiZNxfENU|`^J8b7F~Sr zxGHHo3qu&pfPY>6J*B+%$457}m|p@A8mr9?&>vrq@FlPcMQ5l7esRp#6T>`BkZ~F- zRH)E@{^=sEFE72$koGi)=>IV#m~K*x;oq}xk5BY`)Fm8lm60)h@;)mK{7DNG4tw}$ zXEW&mJp3fX-r}Y+#Cld|E4f~bQ9blZiK3lj+g>C&-|25-iah)xZZj<;yhD-~TH)ue zN$8j-c|h+_kzm8DVlgn~7hgm`^_Lv&*x8D8gwI$FCPBgNq|Cjz^6I-7K(aRa(ph}d z__zq#`t6ebHkA$IjtH6@tX;WfVaDi}CvPcPm|_rwPmp#rY(h*?DW;;uxa6y-@qmINg$1X3TMSkNn=#@!kw?KJz{Xd0F?(xmb z)ou@`8>X8LC1GBual5I;Yr-9yGx&ZnamS~blXGlM1m}YtxvDY#`Tqrlk_%UqWNBDU zC)yNB+nnd-#Ty}%rCyaGM2B}jM2Gm=lU(E}zY-8AAA~blfH7HrY&hK7gcCd)3KCyn zKk|qValGw%f#~U_*q9Focg6RyZi?n6s5I6 z-{ehd7U!mm4l@^CnYBCF8hkBui=9RE1GonYiqMkl6K^%25b2`ZP{8L399m(+ibWJ%k`=U+qyph~kwe*VU0*4=MSMpiYaYb$k(-5&_c#FZk8 zpf3scu_feEI7VxQpBHQW!aa|MB-PS}5z|O2aV6TW!fp4rHpi>W%7wX~{0o;suF${Z z@at~hgP?Ga-AUs;XkHBH$Wm-+-?4VPo2E=a|D9Z|e5{PpVy+|cN~_JIx&VJ0KA3%A z{V)5xd_hI1ST4GxGXR`HY3%>qpUUQviL~GZZH`xWnDT8+L7T%BZ)b4=k%Qc%U*$A? zc{yPEzw}U%m#o))zr1(-$seD+dj;|}3-L~5`%YqM%#SKRW)Cvn=r=_kuGe5eU%sJX02rZ&N?j9Z^oF1mFBOaZzh~-k?vjYj6@)HW zfQcA;g`cdlg76S|muAUzBk?RUU_8|f{KKgs907Z85wL#e72Z91o2eJ}Hzd4fF9&XL z@`f`e!W5(?p6vYAF*5RVPayRa*$95}^TTEefTGvPXZ0_7@v5%*UkR?x-emn1^#IBv zhw71J56br@;Q{*iwl}64L9M9QS_e4yBj_^y4(6Vmy9_RAmMA4No9we4O?|a|3?W|} zjn>Uc@~8+0D{(v0*|bdKc>(s53Fq6A&&4Fmq=C;ge@Z7LiDk4oMcEJ-KtMd^nl6tV zKWrjzj1d(5cUCXX8uQhXx1`A-7(MMf06OJv9DsjuWY6>(XA}Q&^iB0Q4Jv#(eZUMn zhhZ4$*Te6$5P$mPFVR=dne;3aCQXdQNZ>j&F;7lXSX@1!QC5oMNz4dqL+TvYkOl!r zyh$s0csSnlF8>$;R2(aSKy>Keck5$ZM5+@<5>~_MCQC*80gvW^N5C1^N%(=$yIW~w zc?q!h>ou|T+$lH~b3~z3K7b8|E^9CDK;5MTz!zh!^H>%0oVXSG3LY8^IgpJLAbyc| z^!84(*s@|zKx-?-bvs}>FZ$WEX|(lWeSn()6vqCh)SDgGcc)Ye0X#;KN$dx3(P6yQ zUN`~$7pMEnU}rJumjq?qWJq9#Y3sw2x=k^*z;8Z&%_}@9TQ`<1IKydr@2EnBAVdlb z@wo2MbAI{kosb`oS^cp4!ux7;-_~d!R@(o*drpA+hROj$G0S!BCpe})a#ajrWX1M@ zPW>{tq)hWaL#^KhlZCiO1v>ouBTp5%>l+}8FY4ed8!_Ac_X&@@_5~-5=-ws{9xe+U z!oYBz?(AjFS(-S6OJrKps*n;E%lJEmNO5cDeCS)RO|x~;^H0sKQMO{AgzAOrP5%wQ zdcD0H)Ms%yw;#Fqb>^P*``ngYa51ZLs&4i&QIRchrtcT0ds5_Z@;?c66^>knMw1~0 z?R3ek%hh+1Ds}lD&Fqj&ya?H+Sa_IRv)M4y1-7Ek<6p(p-{+qoN0zQFW79DzW{``_ zV%8^-#F4~2PIyp2aFJ*PRUObaf568ibi^hAPo*WxzYIam>Yp5TIj*(0?q3@Tk;^mJ ze<-YXp1Cy>VJ$qSb;0bGe0=d-`7m2pki~&qVQ{EtQlI{&rQNdV}Az-2O=~PR_-S4~9&S3s2E~lc|O8P47K)AWl!wI9Pr? zdwFM=O2P(l^uyoVUY)_OROGvEmokE3KJ=M!do1j623w!Tqg_sH%&h)D0yw_?KMVQ| z9!g+*-J7wvZ1dUAKIz46rR|Tu_LCpnsvLQ0{*0`~BI|;bfdxll2u*)s^g!1$e`|r) zY*neX!zBknaN_YkXt=H2^SC1%Vfrcmm$XL+J5j-zJ%CE{&WZJ*zi4WI*41X#QBU`t z$#BwwawlRJBo)d5GE-?Q3&2Iw(dXuRuO$oMBFx?F`!~LT;nrrB&gBceYPPv2qUI$! z9)|(apSF@~n1gO_`sd<8zShArlb77Zy?;GD&VL|gKc$kUNfQ@pFdw+Tgfn!0VfKdg zgumv&oiN1yz>``AL2|*`{@W)c0C0W%{Ykda{~_4?3nmED`?4D{G`Qut=}E+eiO0V? zP}P5@4CyR&mE|b4^H5lIQodC~Jz-ZAOTEo@Wi8A+#yD>%RftbmSb$%Kyvy1jd_Qpe z`nC8 zRF2l9I-B2Xk)a~J7wwKiB=}_O+BN?saw4 z5e`1o`OvKAe0Rqzs#22uH~mfu`4zsc-1iFlO(Kg*{kM0gO6n>Ibw?@A^wG(G$qz_D zw7AKDbZ(cbui7sO3w_<^kWrmL{DM30L<-_WgYl#zt+zVR;?qPQ#?mvb~B0$@bU^7On9og$je4K$Dwp zI6V~It@vs0b>eE@$CNR+B>L4_=$<#T_e$ibHp8y{^hO~3yhJJ~Ok|j5&+og#@O#7i z!M(Y>4>qM4FaNpS3{EYBkn>5s4Y!OXr9&W z)U-&ojc|{5I4Xq>`gs0?zITwSGk-57h&`E`(fg`e9&#+1x<{h8U|`NJ^@lJOQ}}MW zBj8MfwbJGKl7LLo)b{NqbUo;1%lu|P{_kXY__WHYVtL-Pi2jvjdvn=0kuleb#kFeu zGO}ggMA00>1gm!RGDOVo)V5oUk@4de9$%=(8(j<~z?ttqmYyHop!JdRtH)=>kAznJD(g!KY&uwYnkAd5X+g+>&p_KjcHk9@iSK=t1b*~T_1Dc z6C2UPo(nN&Q2NnqE7(*%0po#(3nU>iuFh3UZko!cpC&F{8>LSDxMxto3)*V=p~kBD zfnQ%OT=e%8pHjq}jr^zL3d)__M3LCFqT}I4MH4KT+{rMs@K<2=!-gw*s)BnIb%H)d zoXIu;;tDv(ov&{76oXb1Vzo*R3q#&!*($?X@w`u}u{jug?{u5+OR`SMPZEkTq`8Z& zd)OsTTk=A_yd=MtZ@=|dxh~HFAT!uDDnMBu7ynefRxTVqYLg`BFJ?0;XUA|lSp`ix zjH@6Fnt{_jtlDE$)wM8x?h(fHBt^}=h%yjq1U+;<>=R!77drmT0md`KLoTPw?br2b zi;tC}Hd)VeVMYivi!?Xi%)K|<5+jxze6N`jnK3Hk51hoJEio)GqJl^=Vl7Em#yGTK zCmC-8RzV!$vfCv@jAhFQR72!5s!aKQJ@8E#aF5c%R^S1csR3M0hkuG!&tQA~ma=RM zrL~QJpA(2CWJip1E-Q|*(Oa=$i~1EO#a+K%uI2G-ii(I^K+U8yMVWex4{rN9PvanA zc{-=E3*m^B%Vi6%5$OBvUrG0gmL{Ihl7$>NPiZqF{B%}2$u1t8A6UUdmgXOX{Gl^^LEPZ(F@zVq-$^_@4NvaX6v~I#SfkChYoiPUMKg9Nd zUa&e4DsVt@}Q|vu<+n?^>na1Ah8{pC*-?J3GO2O<7nGjUhN^|Dv;Mmj?tKa)REv zt5gxO2NKvD@;WZ}X81~86~|P$$9SG6_Vc7kwyYDHyYquC5zW;pJ~-u+B%!FquhXfM zn!Z7mgGLcAh~_RPWFRYQ?Yy2haDRUF@T!YegzQQyw7Hsv&CKqOolYV?dBH9Bv)!W|=w^~&Sk^&$*cL~M66NHVT$bJ)gjD9;Pt1A16&6dUuhNIEiM>t@^Yo~B*<|5 z48v=RzBNGW1yiE?N@=Ug;O^DWR!mpoc6swMV9OYEd5Lnu6p$mN5sAPJ~nfN3e zdSOEkA-BK|<6xt_-hICP_Z+vORfy28M()vzYGX#o-m8ua1#n+5Ur_77uqKq%cjRiI zKD&R@=6s+zBH-lacau^@6N`(I{P={!i-S4SCl!(JW^sozhbY>R)6rt@ZD+IxwwP_Sshbj=_Aho`vvYNn4W4}6>m7RMbBQvK0>M)eo%V8@_q*@m6Jhc{n#xqg=1VCNY zk2sCLt<`;3u_d8&A4I+jBN)4qZ|llWhpZEkZbwi(GRHd|+4X+fyi8-<5z=NBd^ok6 z@_pPE=Oo;+H&RJ<8`hb8GZcVF#>@jQS_W+9efZ9HvdgR!te;M&3J-ZQVx5@BiQhev z58kJ>EfOBuAB4B*-RSb2m+gVWkNX(D*GU)H zWDZfS6Pebzp(ho0SKR*dl}rqLvrb=h+}2^?wk8M*T81Dq-n3-7 z1W!wE&{8JxwB2i4xz$8p*E#91=3c7Q=Dm`l}%Ze>0|+!+gbfZT0D-Pu9pQ* zOeSw;N3|F_>~=OSM~*I#OL-KhjWnT+#KO;}PCD(Jqx9Iup#XGUUks;#oTp4aJx1|X zct&DJ3gkJIiP~(0bFDW%aEi+6F0&vY%(X`DdTa2uLNnj9-I)E!R|!7EXd?Ulhhb>< zauUE`#yy|_)Et!pUExhxFqWPlDFbeayPrt;E zowEPjoe$t$B=+qqMUr-swZCf0LDGz*I(=zE?TyuUdUfRIkOH>F#+eE=H?wg&@qHkz zqoKGU_i~TX-*M}^%xWOnI-kSQ!k7P@1JTuChG6v9=>PnV|NL~bEpF720_}MM5 z3a(Ftp7tq^UZBqNw!jZQYxBZ$TvV^mzdOfAmrhTgs89TVM4e?^lu^6&XXuiUke)$M zLb|(4LMbH$l$H<>q=ycvA*D+~x{+>aB&4JV=^nZUcyFHPob!I+lOOcw=HC0d)>{8X z#SpB`IfYVweCY%jw{5AD9`cEuBWyq;)Di`2HZf-GNIwy#RQXGmDT;}AQM4jjuOzS5 zE-&ctVYLU(SKcQz^}{bY{<5ejx{VzTE9ct_V0+sQ@kT*LT@37Cx@M_sYC^El5q_L_ zJepqtkg^eR_2Y|2SOr3g6=i>nGmv~XUl4`8l`gg!z$pUG(T9g~^s98+YS3f^nWO~qJ0lkOY}?7mRGorEkR5C$C@O(fp9GRBtTp1+|L9@pcKY_m-SBC}JBqk$`_4YBtWqF^k{CLmB9lfh-FSY-mb z8Li%@zZh*wjdK`a_t9VQPY%?*|o;t6yc9 zUux#*8`zQK!w@EpSM}&pe?Hu};C_`@T|k$Lc{NbLHyA8C{U$#Q)sR0P3z;$TZG~}u z03)S2e%@gL2svf1sqb=7LI1GW1C`y#zqPuT!&LNVeL)@Ynh{ae2ku`V#Mc|eW_O8Y zQ{hnlAkuV-GJA0R+gxFZw*=Fiz0Vj|V^BwS*R2^MZVua%=-d`~t|C|0lo2Izp=5XS zkAKiVJfbqb;AL8Jy*j6QHjnBs$^S5IXA)W)2!Ts+SbS<-9s0ELe^*?L&x3p6JmaM- zwZuK@#Q7JA5>u_BofV zoCu+n6y4s{r2;jL444tC{L|2}{ZK3#`IkFS3{UeNAa39TQ-Rz3Wo> zzL)zCZXQ@pf&f#x@X4n1=)cX_LYx>Bn%_%z7_LnP;cPuz(X%i*lAED5RdZEZd`S_a z;H*>Npm0?~L9mWYCIba1=Tc$S9t{{T5ohf__F!6qE*oavZM3de0m`(X)gxj!coR3_ zbf}9=4@e#-(0x%>=lp~X`+E`ByqSvmVYylCNnH&5Ic7N> zhs-hM??wV{8AX#9e~Pt@*px`sleuLcq8K9Qe?hJ`IIzloFN>o0Ktqdld=rfHXGbIz zpLvHL_)m+6=T1;hYTR~1UnEuTvW(g0pQ{$@@u%wdQyRA^E<5pR`am$ZK!A?xVXX`a zd{A*$f28kXj_&eFfn$0`Jq4sEZ6OK@q27~>&EA5qJ_>)w1rf7o+zWuQo|}Kj-2{er znVB(rhg{{!E5aB*S24Y7$blL1Tk5fMhFL6LZHIlS|J5XYh1C?w}Q3sY-NUvP~hAH)N$O2{PFn;sAG-6#D5DwPk@FN3yVKRax`N+Bj~>T!y&eg9xbsihUc*x<3*@* z)yCBLBY)70L?W=kRL5-Lpc4eaA7_O{VuU*{8F^Ij-%5>)^ z?`ane8Y^#sr^J&sQ6@(-U}+?k+KgAaOt2N7tnr($kxP1;3ZTib4hzR<>utjE&)15X zKY$F625two)IT1yAW)bw`?|~t%ik{UWF8h&MH ztY~;tHkxga`wD`z?H#xBml>4NZe(TIqT}1MOPkZ=LyP zt^NOAI9yC~tgZ4@1BtEs@l`D?1+sTc69CFeV|O#=9Ehu%uQrSFWB`7U{#Nf8avhs&G}{gLh*uIT%V z!*@|vSs>&zOYk^Xo#!JA`k^Wb zC`SfNWP$p4rTM1=vrcGG6Vzt;^;zistf1)hcIw*;{`B!3GVIEZ0ECNyZ4}lpH4ZpP zvt2d^GGF>k0)*v9y6oGo_GK|Y)H!cAYu6qA_$!I%@NWzHOT0cwKO(e8fSZv?i83jG zW-=6W9MG#|u<)&Qw-s(FRZOK*acEH_AmRiCdb~KlC5lKh44jBevqWr3$}(B%SN}wA z$WuV*3S8N_KN^&5LON-}f*Z%8i1Wt69{<%^PqoIgP(b>x#SGvO7tH*5y2VY<8TAqL z#NcxvYX$cYX|P`eclg5l7sbq@EJ0QQ5SGfP5YD@|^gdSK7MwSD^oab!uw;;tv;4oD zBDpb5y2oC~F4j`T1nx{=U1Jk@QZz-cYDV4sgDm2JBkco9mZ6OYHtu%9I1$#R)i4+p|RLMVQN1Ef6v1 zO4@cS-TPUM)LXowY%X)QWpnz2I z;4pOnq?@b9YGgPME!@OOX`6k_9lw&3Ww+S8c-xH)c0cH78?Mi5{pa_EI1@Q!^sH=t z5TGn@{D-p8p>OI1ykPpz*%N|K*sZ?+INCwrOjc_&_m-H0V#IPOQ;0NOR44H5?+{** z1@^q=F3Bh8Pw0--j`Cs&+&A#b8L&lvUFkBA2p>NBB@-UweR3vLJ;b><&iL#dlAdxU z%2e7)2fbx%+dfSef@chp2aR=Z5pg&Z3YOO8XhI{mI%sd$zrWK4ze`~YXRQW7gHmLN z;uVbi5IFd@+AKVTq!yjGjznXRJ3tIzMZrRN)-icMqGc-5^F#0jL&A$;GR(<8j*(P_ zETK`_1p1c6Z0997Fz9s}Sd;PyyP9ek9WWfZBq7fk?? z?|aWk9e!*WFu+DziLz@z0CEo`SpVCDWpMx^#b7Z-03l<5wC%xjg*Q?!lBq z60e8WiX#ueBXs-q(eiZwKVD9E=G|xU41BH@jqB6xTxn_XQ%QC9%W-Gy_;WUY7J{tJ zbJ56$aSLF?00HMr3W?=RJ>RAOa5XmjEd2lc`I9$uqo--W5h3_y@PpCq+=lAr28j`B z-C&X8)~Dt`?K&%!4Jo5)N%y}?$lK68CQ-luDU8o#4NOl=)(vW$%?|~Oy);L`#a{){ zpjr1fqevmcK0%*RRKCRGV(-96+58sHxhvL*!M)630U;*cRC?~=T+{u_*SpFPcG&iEIav3{H&w+5r&Ld4cVa+5{$@tGI1T;iig{=*>tp1x4D6oe z(O>>(O7^?YeBaRPZeB!RM5<}$^D;zxeJ~F>h+4Ibf2&@)YT(DC8Ox=TTRg#5plRYY z+p|D8p&J_>%UmiY>&QqXAkLPfORlA76V5Tlqt(p<_UT}0ry_o&2uF6&ZtKz9MJFvK zn$Xor^`9<-L61H(jf41RX|c}!MT*b$Rw=gBx$XQr`U791-VV4W=$*0j193n0 z_u+wQ6VR}e+)AyU0SM;Xh&9gto>OGtGT*tYK4v*4NLDl)cDkT_KntMOy(^1hpXL6S8z43nY3iT)nyZE z26op=TukkU#zkcMp>6}+aFwgblsi3D0ITuyzgUfhDj+W7yjjXL!BDo%3lxLuWQx|G zDL{Z?K*r49qLtf?Z<*^|-AfLeAM0Sqi!+OCP1%$!xxR}wIo5p%Gp1i$G2ajwj+$7| z*or*dn`T~D*GjbRPeAW{EIh$Le6I=C=-~6mJSQ3GbW-%)(t9Sn^5G+U0M}IMzRa?? z_DdW2Fr|vYc?rkmTJ&kO3~pDk!WWk%*WNjz-u@1?!8<_DfZ6y7)6pq*R7J(c@IM1J zMl#t>FbErEs%p#28<35YmBX5B%F#hEy*-u+<>vMe!NJcs;j01^0@j|2loKkEE)V(u zV~~uvqx@%>$8}0S#R19r(Ou=IG!c&)slu;{Y!kyNjzQg)kh%36%$pp{MxmimG&yRp ztYNNZaR9{qEi4tShicd-+h_-kFS53{sazq^ur zoPdNd6|ZJfz4b1<@s)!i^gI&2T-3f$d6V;SH>r8OshZ6*{b;_{2bAochCcew`2sRy zk5FN3Y7neT9txWZyl}LlSO2;R#nj4y{IIFi4lr)(m`#x@W(Gi>prbm7(gEP3br*#@ zpqg1AI{bn1$`ZxHI1-39g&{4yqjZeWI1MH)r zx!s&m@8c6~g%&-sbYRCR6^1YV?+^ha7W85CCbqSR`2sl8;;^kGP_?~;Q|r7_Me;cZwaTb-WeNn(g58QBN?LcUt<+M z%(qCcJAX?GAcW?0r^i0O?lY0QME?vD!(j;mKSzE?doSEi4&s5i50s2_QyT4e5Eqm1 zeBUkyc@xLHH^)2-A+ly6LYtzh@O(_mJx;G^e=tAMoaw_t{BM8YAOk)~H0&|cDx)ha zyg=((NAWxxFtOGa%*7jOaf1hQA%k*0MDoe<{EOrp9byXv*X#a5BL^9nCIf!-?~eRy z^$9=$J>rJPfqDx_w0Fs#5o(oB%p^aTr(3QjFBOqS1zKJdir$!6rdtJNg5NpT*VYEt zVUJEk?`C|WjI^N^$jB9~CF-4$OfV-ak~JsT_2+&x-OvP8~050v^f>{JAYdJ+Yf(hmHGVYhESyq+Bg1RHs_;)0_((l(m(&HeNpeNNL5=Ls9)-nP*c}S2fC?VO#5`8mc};i(K5#rL|t^Ze}kRw z{L&xBk)g%N+g@t)pkQn-m8EUE*BiUSb2sI6oe6kM{aeWYA!zSD?l8Krz)OVH`XoB> zC5cfu+dWRgM=$*fi4nMQ%vX<#{7Iv;xTa=b0i-N}!4)-ZXxE4Bx-;fVX1lII&6og# zT@#f8z9s&;z|G_t5?{k?MjIV%n5Y^k=$abgzuF`;dBweOL?DSF(IA6H{07HNbJ}+?NK?=PPxkb?5kxLC;Lb6yqy}BdkKjPo?;Jux36A z{x?IYE>IFO!yc=xiCp*@yC1$D3+|imOIU{JU+17!B4&I$;!oCd;XRrh{#prz&lEZSvMC;hBV>=@f;03>A{ z=+(BsesA2AA1(65o;)_t&5kqJ<9!+C@y5_)7A&LkZqmiH+ZG#i9`tmz5uS z@U|Bkw|$fbT-N$FuLJGdhC;NwkIWN=RshvD-@5NzzC{I}@L_JU=N+rHPG zn;Jt^d`fObdW+1DCo%-Xnu6K}POWHAO8n1wPb~du9VNgqICoIIpm0`@EOo<4%Hxt2 zSvIgAv2sp1hGsu}9LFf4-FnG^+d*--aKx1BgZ%8}YJ-p1;Z14)?IQuSFKiXaI2S==DT&b?PG1HdXk`H%?Yg@{Y zq~l)s)$gdn8XCa-xkACF%L#}31H-AhZ<553{Si^GDTX!fK0^4=Dd;nlqQD)3O|LF& z>^u10Pl&CVwVX=?J1uX7Kt|I@pot)9EQm+JsWK_(7!gP`czw3J{ylvi zdsb4U6d5B^~;eh`{=xw)b=(@Ewi+279Szz_7>dM{!UM_}26k(_2G1=*w!%K!b zo3ZuHNm<`{t{{O1#m}rFf%YQW&J3EB3NOk|naTh+lttB z0h`6l6NU;P#`mprucLD8$i&}=^Jwh!=%O)wROe(Snr~pP!S{LNi21omNr3e9?QNs> zV%d1su8mMdFi5V59`D>SN64RZbYyrcvuJcEuG25_t`gKQsidCp4m+}XPutTMXgXW} zMXIgM2&F)|Z*O+5A^y>6H9`6o^r2J_7uCUHRh_qYFL=N-Lx&#tn)uUs4$9r4-Ti4M zq3#jwH4HgfPrMy*K`1fWrP`~SjRJ)ptdcp$QChe3ejurRa>@Be&HRpQ*!S&{FqAvc z+5T`_DJvCo2S5%P0NXNaLMfu}FDtIX7C>v2Ba&bO?kimsz!Br2GmWr+_;0qJ^Z`nX z43zxq$c%tEdcxG5?`WJ}in z_7^MtB@hV71I!_*PsMw_{~3Wz*t@B&qk`KOA_F0tY6&R+TW_EL8Q7t^559RLiN@!p zLg1LI*UOgaOD;dXMm~AfgTg+cjoOZQxCWTX?pm6*D7LOL2p5$w5sh4IDSEn=bgz^V zx}HUybo$Y#l#Dov{&MmUB~ifkCTj@57F{6E>u4B{@A@set1J`&C6#%)W;J0>VmPKu zGK~kQ)neukzNAh~eb0pk7SZkF>sRF^EH4Thk z^CeU|i?Q5zE{f5LS{-(g2hd9r%R6rngPMm0qZ~^_EZsLt^?~PpYW??W9q3Qs9b#CR z^0MzU(W3&}TN>!;!nv)wNNTXHe|SMN1e-fXylf|XmzGLfA5kjokFlfAk@|kmp`ZbO zSQbm}J@@F{LUP%gajDrPU3b9V2-o0M5cbFLgP;^0fu{+za#^XI6Q*cQ;B+!1L+Yat4su8;9G3or9;p92fydgTMR52Kf*4svVZi-f~b zQB#}!qvy4{7*cvCW+%>OW~f0%@qhJC?Y%Inublemm`RymCCCvy(o2_ojwkmgmd)-3 zGPhA)4=L_{YVNWodH&a|`sM%(o2A8vx$K$klwwpjFc$Eo)sX@~g)+?^wIZ^`hZMM} zGQiQzlM5vr*DlXuz~`nruYBK~mz@o@p&pj$JW&Pv{hfZid~|?$DPzjm#IraJ5bM_f zw*4ea%zG=Z;){!c}l_i?$RAjdB`Gf6Fo;8^MQE22znnVDWqfvK?iGe`~nL?tL zj$Wa+@Q~%V&hba^pNFW8po?Lid5UnnE`ESnV1h?qqafsR(ZV$Gb~8HO&pUj#Yhxu)?{2P-5Z`vTpD z#WU?ETH_!t#ZyElf7yYu0~WOV_7%9i5uOWihUsJ|8uY3~?vo!4jwR1#;1#_6&KT=# zPM=LfDvz4xU(0uD=(zG@_N z&WHrfC*JBe`;kRt<8NFt|C~aIt1XhEv$1}CDwvHSvN@??^qVu}<$xz=qBs%gO(X`@ z_5FxEhVsrwKO?|!c-*{v%W2aHGusEmD7}b=O`|I5Juf%Y&7ZuaHWk%n7JxxwXN)gj zwdT7WU*ZI_48AauQKtJ@6Z=a;P|%kTILWF{M#Pv?K5xw0W5SHwGuQT)Uf74*e?chC+3jP9 z0DB?xUVt<(5V=^-n%G|20OTmnW;EuRKK>)cke0mZ<7iU7&e=v-y7()Sja@y`=@72p z4p%&(UES!sE}n9&K=cj1DLIpi!^CjGrQ5#lE7j8I;6-GNIG=q%SJr43J^(#u3Ae_6 zRzldOP3y{mAP|c7e9f?1$P5{NE#_X_1Wt`4cF5b3gkQ@H7F_GfE2t z(o(mQz;ee{`JU;n{zgGbT!wUH{Cx%WKddN`UJ`rbz5`vifBSFj)w2Uhs5|f*@D@7I z#KSD0gN#~!;w2c|(%%K7xJRz)H#y=CXY3P>%&=4~9-aPbPz*ASj@lhC0TVBfuVu5J z=P)*XG>krh3-T2;O$2@UyG%FZC~@T1?Q(W_qIH_cjqE6GG=xuqSApFfK7rkz@jUDA^ej*-Vc&9|>7Z?)xz? zzg&Lq(GKf!6v_5d`roB=BfzJxKK`6?y4~*07rpye^^6p}C!}(Q-@R%uo7e6d=vL~p1TcZQJ%db$*`9GdH zfR?Ie2W~`jeO_E|AlOon<`B1;5lP3+hBBw>t3<% zz!9Eq?M)0)^rqL4m%&)rR${hw9V4yqlglK$ z-FY#2Jf@7fBa?$c4NBJ86+Tew!GOwVTxL3$=ImB=MNfKdwxE%P&4nSJg1$FROp{?s z=AgJ`%hSLMQ1$@LW2{A*EN|8-or=R@oFO58;LeZ#-9k!k`oxil@I!) zxk`O!x;XH%2j2D=|4A}FO4_q1d0Q68HzF*T>2p47UNa&BdyY9>wi8p3n=xWVgV|Mx z7R^IWwyQd(U(PkhV;F2gU1snWRjDOo4hS}{BAwqZ8^fZMKNTKVaKKCMR zS~n2I4u7a!;_IhK(;B87`99)^fNs7|(il7VlBWYDlqYB~mtixA`*au)EM$$ZH|1L8 zY>oPLCRheKzZv{=p^)msCN9i6>%=ra^(dHV-UfpM$8)UUTIR_Lvr)mgpy%cnQ!YOZ zwNLp79F{iY#ks$w2j>uOX{(d;>IhIv4Tc zW3o4{XUAjDFY6f6gCM^I-TuN=7(?^nXTR5(ks}WyV9}4vIKa{Kf@*IE`4?>we%VLx zALDFEA5qfEJ6mn&q&|Wk(Bt~i#yjYy=05d(;oVQTtx=ZCRbjt$lK&yG9^D$Pl9qYq z52*kmv3&t=Z6-Q#Rmw0wj0*qgfU8YbSnoOgSNiu!RMxDQP6wQDri~k^wg71wy-p7b z95Lh&X?Rzdp+o7gq8|iBE|Lql)emUv-LKw9>-ai)akK(Wf%YWXO31)7Nnko6X0&E> z?1Jo8?E?a@K_CMQkrHGWO-=jfZh147AsO@8^0$QkTp&y_|J806t#(&ol25Hati(3tckkJDr>W?{gBe>ee$n3_=)bmZSQT?MyT_>ZjZ0CR_D1V|` z+P>hUqyKjnFBE9ch(KA4Fv4m08(0?le&X##thWVQ!x;A4KAxXSE z*BY)n6)18Iun8w-J9qN}b+0-GPV0U0uO5?@LT!jBw?%w;A9V8z9%Lf zPD~8EHDMA2olI1J>Czf2b8Abv-8E3y>ST03up*)TE;ncj3V6|p* zCI6Wo2bc4=Nn=hBeO^KctL;p5PP(3E8ZZ8QTXiydmk0kqAMlQX0u*YJ|G$CQZHwEX z@;#m~erc>K4?~Ff*XjbL9bKZK{&*jOX6`XVbHiVeckVzqwyy9>2Ai&yqk2D@kB%Z^ zJd%EUJh$)5Hl55r3is70p`iP>Ok&4@)UEZ!M&)-da&+qs^ka(G^$Nvi{L{V<0Cco> zu3_KlKScb1EaaycwJ06ihX2kG)BUpG)Mt;zqsg}Z_K8&H{OP8b?SwHH#ggh>F7H`z z*bW@2wQ0j%n_U3N1E|MPGzALGdwy^m+?16fnIYG;5Exaj!7WB&b zEQW@;&U1R7SIwy@FHc^NmNTFb^nnb<#yFZ^7VTaNjErVGg`v!8BaVWVW0o=)XU-E7 z>gE9-TKRR-vi>mb=di{Yd8gw_!Hyb|gg=iRdU!L@;SnxTTFvoz2g?fIk&1ZABF ziA{n@4lBSWZ8?OJfh9ZU!R4o4$PCebF=i{1j6mQD^}R$1c&s(g-W>rfFCm5aZNRn! z@r+8;cB4zwGe_(_ws)%!w(_04A;CZ@BVFgT9`uBAqwTr`P!%3s|pl1~RpbdVI@o#;ycu z>ydb~adLhG9Gx#Z2AlE(@q$Ri{lP6wr3>2V6sF+~e&e3|ICOx6{K2to2x(yy4tFunQAQ4 zq<4w-g5vm$s}0Hb)r;=1{yuq|gqUFizzTqGa}GBUgL;^0agXV^U)Y*0am73c%Crm;}AA#dcYJ z%BHqNPyF4i7|p#a{`FmfC8i8<0Js1tqAcFmrm~T16EoUtc)Il4e7|ITl$h-=*s#Wl zd5psLC{)f;&p4%3TJ$Gtf=sKMV2i77Z-G(%+3NktYO6lFv| zTz_bWm8ugy^O}<3=HTzo91DjLedsF1Up!QYZ+&i)Ta3`_VM*j_){s&|_M+GSM$}9tjwIKdjDR!y?-ZFf?Dt%mI6fwN7jZ z!GT{i=2ILPK|4PK^r?f7nZ@_EFhyYR3Il?k3V^al#{S*)%C0EW)vBzC5LH;GfYxfR z=uqkIqp2AiNiDzzR$qwQ*GD{5WVQL+IMkf?o=*$$k2XO74d@lcr>+bAiv2mZr@qvG zrz@5LKE|@oMcu@sC9XT`R`2Szt{;E{63RpswLM4K_{&!+RVM;dDwSNBq$9Le5hjex zec777VEJGM|5yJ&L9TzJ#cBe(vhV}f{sW~f2ePBr#zaGI)7R}~Qr!>BmSbJKb*Xe!PEn?zZ*l6GV>k4&hE_GZ3zF5j)!(j4!@s@ ze}=y=I5J52OY=9k0!yo`#fbyo7GeKU-~+^{&~s#tT)^L4GjoD!6sr#g#);Z4M57H z=1xr;#KZrk$Z^y@gWlwpaGyz8>F7xPCEfr9^RPfgDC!vmh;!W&t+ z5na9|Vqf-H02BAb)?UN%XJ9&=*ExFp{DpJ3VjPo!5|goy@30HX*rt8;`2->GKIG6` zj&+stq=8AL@1LfIBCxECc#51nfO@(DOPzC)jLR384X~Wsp(7)4eD9V37I7{kuRSUy`qg zCp!lB-P(q9h|}Eem%oWZcS=L!y?esRjn_h*!bxRJ_s|yR^ajLC)N?0{W&*MA|5~!x zX5BhLNgVxOpaaU{CHE6c1UWNzj$c@P*m0!YSq{*o^+CBKPz=No@#&~> zY~g~Dah%|Lg>!pmLD=8WzUz9!A#*Q~R8$p3f3e@YWOqjm7wUti6KYs4)?Ut2x z(8PAKj1&#CTWq+*}NJ1?3q;VEU zSG!?OComibu8|kE%Yx*&P^p_?6Pz(%Mnb{e$Ls{+)*%L_M) z5u)eIT%B;C*VGK?qEi%NcFBQht`z-|m>+V#Ipd;)>0fQrAktsy0`AWy_6!k`DyzK? zI9V3vpLcKC-OP06O2l#DcXEh+I2awPAF$9dN_d?U6gUFDDF-23qE>EhViIMRPWX1= zk>>twv0*2XxKaSQ&<0n1%5$5k;y0b?jXVe?>mLk{sa;-Wa|r*t=0Fai||SeZz+Io)P`eG0Uw1`~W1xwU^p= zpgfqQ(JJf9-RGxt0e0$~JAolUXZYExqU&oKvu{8~i&LDd40DFmL#jdGzmVC1hl?@J z=C`oNB0vUvD8uc0dArJ3!m`MIC~=8BAonp>*|=@`L?b(}Alrnl&Bir)qUx29_*j5) zW>I4f)Aei=2)t1mx_FX4Kk6EUtM-n>IXic;c}s*0B}@3fECBfJ-VZ|8=}4$vjs`{> zWdOk{1_%Ecxoo(-%8OuOaSs4D%OlV*Ksm~QjBP5at;NDw@92DmkeiUy{~<1@R=0I& zb}CEmM~W%k5g;HF_W05Z3SFVMVCtpKP6`SJa8wQn*vkbj?X=iX}$7`J+2zN2_@+XhFywq)E-wAyKy7JdpO z;j2LRpErDWP>0ts@0$A~`u3AJimyCd>O8B`c`hJ#p9{n?@bT6G(LlwJOwz5)Mh`I< zT+zRr3iofYmxLW zfKJnnECg`RU}HT+!(GHfS#zxL6oMZE{y&K}0A+Z!5(+{4TA_d5HaXljYO?yEKEK=s zB7m>IvFm?#)e9k{qo%}2;j@Znf7hgwajLfYjU_BD;Wt0SbCli5oNdKGgVU;3^$QB| ztZGxUj8q7j3klvgxZH{}8dnzVx(KO8Y7=i+p4gLi> zcX2c>vmlt-?PvQ=SG}?^e@Nrel9R!MH;`!x*Z{vQ7NN217K2eIpyZhVT6+)SwQz{X>n@#^5{n`rmp6QcXa^4T+yoIax6u2}B` zC;DD&5Ac1=Q2H)DSrc97N3U^`^Z3P?KkZ5-vxzuqrJosXAqMSw@0Iax*ad3W2&8p_ zX5hYBXDuQ_Xe`?IzLW_Etqq9rj^hr$k346zzlYPEbmT8Qy(l^o#e3_dao>g2 zuuJm(TXu*WtMISYv7WU7#mpEbV{z>$Bu_ReIEEMC)eAFDgJ!i1LS(ndwfu1+a+!@; z0xyW}Y!89z+Lwf4d)$no3Ka_VEzr^&1YoZyH}JcUqMxp*5%=92D_!W?S1lOWH)%~H zZ*sMnHZZ2;?evG}J5dj4J{3F;j8%x_9{f6O-T}7-!!T|L+JW_1KLCxRMKc471Ezp3meDlnM)JSwpcz4cQ=Sit7?;`{q7}~7>k}4qOJJ9BOzUA97V*~8u(5l}75Y%BJuvA&; zHhcLG26OdD8LYw&abq9re|$rGn1N8LcUY|5@@TPj4&Z!92C9WW&tkSw_ac7;czH{Z zs2V`|f7RlV9{2Dq9C)bZinAZLM|uHOKKVe_-qSwt*CQ_q7HrV*+ zJs5#Vaw8GF+D;=J*oafFwoql2WO5B6T9Bto-U6)(>zblR+w~ggLH$R(T#LP&C*4M# z@cWMVZa_CX;7hKUtbdQL*TR)xj_XFX1m!%0U*ukc6zNVdB$SDypRW1in}6Ti&SsXY zunEGp_+ABS3V7zOV^BU$sONFI;6K$#XrqHmUtG+VAmnCtp(aMmos+u^dW49IN}V*S z6GUx_L57lLTJs8nugG~uJJA9P6K`>ugkvkmyGJ9`%KsOd-jh*qO_RZFRW2c0tuThKtYE{RQ1aE zds&%h3sVK&q_O$$D@X(l+=63;5bD=>K-baf-)hV=slg#Akssdl?OtSrIkMlheJPdu zeSi_{Y8?!rde z`u+Vsf)P$OeMU1YaX(+Fk4!}Og>^Vv{C1@~X>3EtQHuLVgagAQn~edRQ3BQ!W$oUb zs>hQTpM8J|##LtLg=ilDi*&H|^is3>F_7L1`-;LLgzf#DcX;RrQ%5J?iUcOr|2TTq zI;7>HDS+3E@lQ8k2f;=s!2JsptG-aXI0N$K*aq7jaOsVY{8gAT{bAC8-&G<{Jxy?K zlW!u4ht1Eefbqj*XubtU8W73=TpBhln3PwscE~`RqPyTZT0j9?WZMCZ_0P}PCi{u* zvJV1@B$b7ayALv@hgc_OlyPCSmWE?2#EPBuwllA6vGnQH{}Gz>^co%(t9vORSO1x) z<{8eM+7T`53f*2U0iuc@L?M1u-L~`W)NbHt;smItP zKr!4iRsKQLynaFK?2sM*XV;}Y3yt&`snqyxq9=*Q#9Ep^G@ln|AvfwiPE&)cW=YTe zB~IY#;DWzslEi6=7--V?sXvzQlz&36S0|lr-j7743Fu{ns!|<0+$gdzy41c;*Mz9_ z`Qg?0v4qxtJLi3}-KgTBDP{+n?t%eyQXHR-z_+>Jzp{~pK%y)Kq>RZqm-Z7=h*bVN z6sy-%Rf33}nF!s^y+yhtR=|Bt|0r4QH}PBxma9~89<3m&(yzX6&UtNJNpck0uQ5O( z`?V8m$!5{9xo}B?`q;sw7xzLY4(Uh1;qEI9InK&()AkSQ`3}+bHPUhv{mobU84mvZ zp)_3LEewAmN6N}sVttlxYj!GqZ|IL_?xbFqNZ#JNrP-CTX{TgGrm~m)A?V0+t&w{A z{>g7&nY%iawr^kkp=$!|tj+6tkrFW=5V8ySr@s^_!%Ftv05v_$%3g0AfUQiDFhGA- z2VR^H*EX`7C?62?JI|?Uf`NJUuSKUjM*b_nqtD`2@W!CU-3j{WTYiHZw!>^^7e;P%6JAU5wDknV0E+s)FJBR9U zoPA&61;JP@t1n-C1`(qE-qDO}K3M(vJs!{^JxasCFF{kFtCUg1DB611fxE+CaN%xj zD4AXZQff~bMh}V+7cMQaxD7dDac|2#ESyQu%ly3PX2w?x{WuKR*(@(Y#^t)vw@=Yo zu(sIY4m;-9+Fj@vM6`_FbTWJ*CfLD$i|C1V z>IaN%vo3ml5~4&n^oW@u|Y&auJc zpiK3pME?-!ASt~|KQN&kdK&qhha<@PiK3s*NB?7MVmA4Si7yS8!E%kc)XkZ>(V9eC zSnI;CiGvRX(6p(+T|DU!{H!DZT>tFPheSs5x0vQ{bhn%3V1gthP{w}|phm!M7UazbWx{5C+gNY3>}^5Ct7RvXai5#PQ>pHpGzh_M&Afz3~Qk3{$RK^XG(I9re? zV68FvOoRn;iu8z0 z#^e%gC1D+tJaifHI!)I>+lu<0{&U;lL~95qo?5}*DU0Upk5saGR%5j|D9G!>WS`X1 zwZ<#4$sueYAqfteBdSaNsWL-dUK}{Ho{raDqwcRjRREVMHRBWNf2%XVnPX`;8RQFk zYr2%z5kQ#!6#Q-b?bQ|->irzlb*d})h{C?$(Ym3IulUI@SqmB?X`&pBCWU?EzYUNG z`&K7MJOpS^NiTdFNIQa%`q7*}#w+%pG%I@5oIshvHW^pT#+Z+C}meOjQ*2)BRk+5 z^f6c(TZ;Sm$xDwgjDbFKnYkAMjFAhbJPy+ymC>%Yg1eNup(0oul%374H50g_r@Cuq zn~j+#JAOZuXnm)CjJi5{1Fd$1pN(K-VShY6zopo+x8|c;PgTLy#E1lG5(h=Dm zjct0W2#xqCGHQ@AX5{YGq==cE$XP4OE)=Dd?4Q}+=MI(=mJSiVuT>VBc3&Z7k8sGb~yj=)hw6koMqbUFAtC`@z6 zI^PG~+fS7M^lu-;zeI6{I%Tg^SM(X1PHfy-Y2%T@+{al~ZOIomv=lpu%FZ#YfvA7gT5cY>Yw z7-1-cRd_U=7(nN(vHTs7@rHnyLFA#7o_xZ6B*V}!_j%^;8Sj&ylf$z=Rb+2p zU1YuM)6Zr#@Y580XcJRZwg2RcA^JAqt&I-$q1q<}zxRAQ_I4GsVQ{B+sW%sr0ou0*+N0q1=izidScG0Bf4RTpfnZ7uU*KtuDg(8Q+CT0uTO*Mwef`OzJz#<7 zYQXDcXiW6Wodr4UXK=#-DBYG^lH1?Vw_9voFR|emWnVI_@hA;r_M5ybv?X+F6>4_| z=So^y8XhBIM0Kk89+PX#NFxYskn4vWgv^sk7^9NF2|(SmD6UFtRkTa(fjA?oQgMH# zB1>ERV?fejAz0uI-(L7t2$rOlqsgKB-@B5ZEKE9kP#V$`DCPxQ>H<*V=R~ayp^T!s zlYGRsaS-)hop)-D5~C_Lav3NPhI%9#1mb=d%otoae$;7F-sdSYDa|;(=u2_AiL@zB z&Xwg`zRYn5309>K7`O~bh4*_G7{L-+c_S%>3+}l~x_kQ>1p{+O9U~jNLWge*pUdBY z80m%5J{>%O$=YQdjrEQ|OV%N_^ha9xDubl7YI^1-$HC+4M!KQA%J5}k1{}H4z&LFE zQk8S^4?G(08A~wOc74tnHYRgQDIvadq*I8E_P9hddD#-XB-y|CthSqKb~8Ao8zo+l zXA=nmh9#G!^@QZAyaMwvQ?fT^lh_6sKl*c0>aMY$kHI-$I-9fx;2z#w z?p@aZ-2fy=Mh@X`Iq!XaxMuGny=#Q;Ob zi9=)4>t#73j)%4DSI5~6@A_gu>lJbJ!nNfjNm8onmXN~~hFX>X6$=OcAy2S(A6H>; zGk<+1;E?iJ7BORAvmdqsMEJ5s0QbW4T~VZ%tatNbuY|_KgM=YG3|=J>4nJC_Il475 zy-^I(pXWj%2Yu*Lc>0wzgBdP!xvZ*g^& zeuUV!-rl399~K>R@ML(Ml@2wNbrfi>eKXb|;%>4Z>X%5b9#DBf5u*#&jx#hT=**q; zj^QY)$d4S$dYD-Y?ZAGP=euzNC8{-$;1>lqQB#hwUp1;di3L>Wo44a?OA(9@Ic*2? zA)s-9nE`Q(IFKT^sv~(}0m~$)tz=*g^a^yK&fa$2UAMVIzO#cH#_1wq`a6!z{_5a| zI%N5@$K?9r1SQKT62$zp1qc%qWt8it#xQb7%ZYXn@JM%x&rY7Ti8Ab)Q+73%y06v? z(GYs%Ii?U@n*HXvQCm-_ud-~RZhOFC%;O=)|;@m-th>- zhKVerd%*&NSBfmR{D=IjA%;vp4&3rg_OX==sTOUrJfFOQJT3P=?L+c;9!xk4`$8mP zI6{(y*`B${0s>$8c+#MO>W)*C$ujelx-yBsBL?3-m0I+a{Dfg)#t-Qc9d(Q1 zhv@rdriux&K7&cC;$%=_G`63-iedZp>1-d=nIJy^YsM(137zO^^9LcB62SD0UQC{p zy-gkogSW|p?lyU7goeJ!4}Bs-fnf>9c;VTudx4Hc5M9!ABT{`ROaSjq5!bn|n!*Wm zFkI5SCpgbbM!hb+Hr-#oWuuQcJ%;W3#!#hmwL1(@C9cGt+?k%`+;TnU?2T>pax|NI zkQ24}cebjB`6Hse|9lqJP^)Kd>@-$=r}vy^lZ@M_l2i6|#v3*i_?)xQ*;gfhf=7-? z{hJ5u+Or07y3G|?Y4DcavZesd;N@-NNQiyjuGQO9(V=75j-2lnB{qm6whr+f>>`qN zBOx=?7^n~_E+S8E4#>y!TPGP@VNs^U&)O1GM=KwEy1iI&|0!&(LFDH{bbS|z>5eY4 z{si}v8yU`qS*QM`$ec7x=TM7^x8H;@CB?My6x1Y=*HXw+wh6e~Lev?99xq9T>wjS< z#mqh#{}lPK_?6?~BEksgeO#M4Nu+xzhADYQv&1WjJjxQSd*XF>TZ7((R+W}TDLMx; z;7AXOEts4g_`VNq)QDW%9s16umx<@sC_6w>2URt&D9(q%jJ|qaX3i}~QGIL`alsL> z+j;~Um`M$7#(|FRXNZ>$K65lw>3met7%cC;;>+O?9*6I_FuYDQnT1Bmh9U8FQ$fR^ z>=gT@;X{O911*AZj{~9+PBL`&WD;~I5=^3t(tp?i`^D$H9ci4C2^dukM%TR?JOjPuFStjpG~FcL)rNS>@J0`uk$CW_SNd{ZEq{ zSski{5D+?>?*+CyTiYQgFsdHLdyRz>?{#M1{fVC z4@!K{EmyWjfg>(=a0DI42vc%Knk*X{kBo-y&=S!eP_T4bFJPa6(BQpqmg>uk_LBAI zk^Y6wYx%ViHYd>74~z$4D03K>LmWeJ{qBhUc&Xu;!h8m zB-|%~iPWm&AObv4l;zaY|B^@aw{6_Ai|mXDo= zKL)z&YfFG!uH0?|Qop>L(+za7l=OZYEG)OVq+Iwmmyo|!@n@oIl)`)XO!<|Yg`NZS zDr88Qbl(`*q<`yo073&T^9ns?iYJAlRukFiS0ORl8l$i8W|riSvGmUYVmiB~fp-O1 zM(40E<}uOnWvXL*lrAAj{Hu*r6o2^}=I;aa?=<}HyYkdC`7$oRjKHM((>*@(!L;zM zD9YbHq8Fz-9M2)x-(D8{DILmc_Ga@x^+`T_CWHI~TlcV*5+tbW3J z3QW}pN&c8lYE@NDzxULr1Vs&)Ab+Cl1Xl8Tck2!WNKYZM=io;}m3=u&Tpi$ORP09Kab#**B z2fL(qj5Ni`i6JaAEfq<3qsj0Qu#jPXKJ5GBvKO113QQ0dt#nAp5ZduV)g8#YWwPeH zUpM*WP-5}}D38B)IPZmD8m{}J=*0*fMBo$Z;Sgd^O$zi9Ty8HN;} zgQRtVceCq9abB`#&}iIPQv(Py@Dhy8*T~((hL`vBHi=*tLbH#?3f^qNrHQW0$<{Un7g68963R`TOl%FjdKqdg zpPh<{`&})M!z!D>q`oMSt)y;OWYjV*Zm%gRxBor5^xz%Us#@#K4CEk$KPS@7$$cWs zTjcmpM!ZL)GtqDjqxTwlqR6cC@bFiApL`_sUY{Sve%yrK%X!w0!de&An~>}Y(H4Tz zuod}xCnaPS^PWS~dp1|G+dmvN6Q14#ywzh?+u6b$|EzCd+o6Vs$P?T9^XRvn>f6Vv z;ge;xS1_fVIpwudY(MG`tNL|cjK_(-1R0ya9&wc~d9ZW${ul_OCaow>3znc!SCGT1 z_=)i>jwP6oNoK|R^T*h#bqBs zN2&j=*as_b5zO1*E?NEV6rm(=V_wi7b-|Y%!Y-sU3}GK3puv{hT>zfrq=j4yTfcqt9PvjmNc@;vO|k%xPMreI;AsJ+l3bY@_uGe-_)|&E-9UQ}(S%K4%MZ<#s7zGyk3f zDVE%>skO6#{PkbI;{@woxcw_Y z3%m8ev9JKc_6?X-6`G_6)4lXLIqbiSA<38yeQ1e-*28;RMLJdeA)7!OH=e;|k|h%o zKKpdU_6`-*bmx7?c|Z!7#uAW>c6p>{QGx?E`LW;;D=c3{isB$Km81He=8dM~VMT{Y zqu_g`!;kX^(0V_=RZ~-&kE2t}z|{BL5fDCG@Q>{pD*IRoaonx6otN-~EQN zA2CoglfV35Z`-{bAc2;2HAK$m>HqMJ!v4|_l)Y;~P>(RI6vU z$%a$M@X-+R-ajitt>l9RZW$Ac1WzD!WU@Y(K0G>?D6lo&(;+#le`^StP4%K$#-kBw z{o1af^o}JQ29H;<5Y}Y>b*m1Mmahd65U9!_VA@Bc_#aw}FX(lL9g7XZUiD z!6nmPbJQzFV4DcEQXcWYJbbHgW|-CR@kx}kHFo@SkdaOqbi`QGQNCOEq(vD5bFR!m*-&VM6$*rrq^l10WzY zF|VcKgZgU1Q4hiX+coesh&PTBf8&ANWmv9j3&rB2_l#ml1unVZ_d{Ts>>XX8kDYw9XuUYRFrK20h3wLWzOBq+l70hqWO4Z#B{RMXyeYN{L-Znuy+`)vYkhze-Ya`v5i0Fxle2v zCA;VwBy=vpY_Hl|^dZ|>bTR7prs1qkDO&q`jPjxfHjSgGm)_=3s614t&0hm#M*pU{ zuuYdYM+nX31r~AFZU5MbsXdp~S^zEqF`!&D0Fv?ZI9xUxm7LAoVuc06xvHtwF(!T0 z=&&EF55Uv_)sdVLtb*HioXR7xprF3*wcD}sXxx{{qV#5pJLc5VGOa7>b-^2p-;_Bu z8Epy3`KwhSfVoKNfP+T5pQ)#c8+B%_HYTtM26ohAv{_(Zb6Z?ry4~2X^lWHkI7ZyK zg8)+q$7{dS#fw{T6z6NHL(+G&J6`>x+0E2v!4 z76o*4)t&yB7-75+nR+82_wnN^c&%k*mT{Cy(0`=uTGa2pqjfc|LwXp}Iz*-CH01JN z)=7-g@aujLcM~UIu~>*LN8XphFC9FeEHhp?TZ&SR_SXTQ*x4S`x*jl6o;>>^;Q0TG zLA7~;cTa{N=Ap!=TVMEISufz$(47zEXtlQOv!;n1DIPoh;f<#EV;WIE%B*eW?BR~v z*UD;{QkmkoX-ZP@k-b;GiYbXaa}O#5K~q4S#e7c2h|YsiX`43$6o))rwVURi z)|}zLzsYFS$Tc*pxN|n-9T#wsJQuw#JJ9|+4ue(l=3=7hH!!GJ=@;E2$cjamL`$$r z`~{TjKBKk-Xd6fYI;(V^VtTZ}%ARWsLY72Ru4C5Hm|C)tc;3@kXN<%z(Lg_|g zpAF2yhWBy(m|ksEHC-3YIP^Sz`S$ImRhNOw5!Fe9$H`$Q?dItJ4l&3Su+KubZy$E?(y6`g(~oT_MYH6Gckk&vkFpv1#2~hT7ZE(F1Q+hY+HX2 z#PfI@(CEJDKFavuUQ(!6_huv#IF7+-)I&kTIrZU;WiipV@gb{hc(k8ikko28^>=hgR^e^Vhxt87&2)S>lOL#=Zr`wF;8aNB*Bs zU&zSESug%!@#-&R_{Uols%Osys3f<+L%3&)+xg^H;K)nX1=V#c+#9?2{ZbfLKoaHM z7?q2a%-Nu0D);@<&aOCMAHLy^-bC6KDSM6;Rs;~gn<_X4nkY_2C2pJA zjr&Hv8$oZm{82WgpfKu+d#FJ>_4kdVfa5<^%f|CmNk)Lk+lD<8DXBmF$R)ZQEkmZg zTVfP&4V0Lk@TDupbKO*Lv2~(_*Pg9sMwxU)vveC#N8^v!c!TXUZFSdZLpm!Z!J2PO z9GX2H5343}8t$Uh5Q|1)cR5EeC(s-3G1UAQRa8*ce7v0xjR2#?Lc$VR`HU7By`!);cHBXZQ zN7zuZyJV>iw|V;Ib3oQvD`$%37A$#53V7U~vop`guD&v!WJbNWX7l1{j($1~YEd&( zXb%Vs_@jj$pdt>piRkRE0k}cye4%g5Z%PjSp{^*9krk8kf#Q!bv-54;YEL@d6#=4$ z7}te0={=55COTI>+e=E`?LMr0V?XkDv0Kva( z%$i0W{U2w55PBbg>UotkaIft89$zy77Jn{mN&bF8yFyD^PZ?d&!3Hd<;eRGhu5!f{_T7(-tAYx zQJf@iee~dxNSfr16a&4Kn$)x9+{m;Ceti9B6Y6F&a!K~GG()7OcVz8`Fw7a<{IhW{vW#vxIX$P1GCilVWG!^EFHX zR~k^Zi+{NN`uK{P*qYrqk_^uC8|-OYrmEx8cmNJx9eA1ZaLDV(QMQ}=1g{=gk0F@+ zSDg~;`%Q_ilt|LIk1(lp7l3DJ`$zFv)~g3P?RNy&T(Q$5x*qb<4n&k4vgN-xqcpGg zII$E0PwQKe6CiEdfQmvOvFXlzu7z`TJW0X%Ugh&%_V26bylmU!9U~2Ep=7C@WCA3& z7f&T^2!w5D4Jat|b4pT!u8S5 zz2+Wk|H%4~*|ly)pGtZcLc`S1q26hLPKt=jUBZF=9vRW9LW3vS4GheZTj~Eu_ho8Q zn=II2w-FoHh!}8-pg!Gf52iTXoW8!u8$9xlWV?nV>*jYP4nkF{2fT9lLe!}CjYnp6 zR}qdE&T6Y;Ahub{JxJz;AB35n?)QB}UU%I|O-Mbpm_BlvsB|W!xiq%RT$0BG#{;{}fYpreL`TL`2d}N;k z7pM03QqB1-;MoS}D<77V5kD}{o{<0WqviF@Gk#8HlYCQjjvRh)sA(#Tzrrp<)t%}zHCw2~OOHP=U z|M}L#_XUmm>TK0j&wk93PEf=8OnoVd)i}TB>6=j)>dG`vTY+{(`CJ(z(}}(F)K3d` zSJ+bcRwA;Z>Y|KEoPC%*q{6BKoP9E4uB5dWWkeo()XrvM*1$~p=}Q<!^0Hp#z0k|Fq=(6Rde&W+ zTI43XIFR30(%Sp#7RjkwJy@hyH$H@Y28D&19@8>!kAgp8>W>rd2&$bx*F|ZrVeuYmc5gP5oRAZJ>p|5N{mG zP+BnuUzNkpWnB#Bl?@Bdw@}GBZ{)zt$d-P^A}uML+3lbuwdQZEP-LLOcL~|D$<-fw z)p?djluUJR_LnxD8Y>y*xXuo$Hm==tDX8?;K1SY1Me>la%$l@DJ!a5o-jb88d% zf0Xf_Jwn&?+KqJjR@or=cdAm1xgm~O@^o~n z&FS_ZQh4~Hix|i?HbAL@u8ov~?z{fohtV7%NM`p>-|`6#t|s5*44kVMm1+v~3lA%K z(oFmpUDWci-cllxoX53S$7y|@8*J#jKRa#C5=YLAc<=i&(KkvJGLRG2Z|UthF#S90 zoK=@&5V*h8E`?=@;1&Z_qmM=l@f&JrSF?AdYeXhEL%*jBZljC2&f^XJ4VC(}PZaPE zbG(1_(n~cXmd{C@)O61{GiX!NQXKg=|H<76oOg@b$?!8YoxjN}jk6aghLVal zj8yD6={m2`vs|5Z`drEJ8y?a$e0mwapGoE%VWHC+2bEHg>art;Na*({npDHfY_4h7 zH#Y1U7#Ox}dHPi$25v{AwG=DwCdS9#Sn+n4-n2Rn)P0`6sk$iz@_$DX8&mgH=;P0^ zh~I7{vHGusBL9mYMLE7-FqkKF9V6~R#m<*wb*!5@ts791BY-O+!DF-3>6?7HD%7+A z`awxopj-RQIoux$e!XRoXp_hr0K;Hj+7AkqdGq`>(yspZEWr1oQhm9$`)Bu#Ai=(W zsiuyz!28LB4lHJEX!JkYK9?_JRn_~nQW;iYj$GH+S&@b?fRmTH#EXhW9UyfpJE89J##|v_Vj*&m7d`9dW{`<4rEYj{lyS$OJW^;IL z581|_6OI;Op=!@xrJ76C!slL%8}DYYoS#F;o{$biy!1L#(Db^hJ$|fl=S=ZX9O#*Ccg#gIdtR$a?JYkA2+cW(m+EvsN8t@#te`u$e?U4UbiBv=~)k9eXI{F}rg z(?!ogV9vZS2b#=@m*Msao;+7wHVvN>z4{ZzPOcsVGQB3M6W~&a@Rucg2CgS*Sa0zV z6nO3vUJm~zBou!$ zaGd_3Ij;L!@_Iy4cxFpR{>g)tsU|f&2Y$PvMzCYj=W=g@PnTbuNlP4-aZ{sEB8@F@ zQdB`yw{Db`a*R;=(VQ8*Wlv zyFm`^x~R&yzi-TA_h*%7qO#Lsi^@SF=v9ZB%cfIs!dTm&(Cgfh-8k}DtW*1 zI$jUcgkw>gO~lsUzYG6;Glbx%pfMN3H?7~Skuy0YCnwuv+^7;${XPBAQDHVk;mE%w z;;>UJ9LrQ9lcgXD|4VzOO@)$VGU-~0_H9nX;P|v?ii#EV2wQ{wAC^VkctsLonZrG-!s{iP z&-&N!i*u_wPfkv2q}7e@d@QainK(H;7Ws6*D-qaO2t5xq`0d-zr`;ES@^^oREdSn* zbY*$-!>{msISIO+D5?vSSE=$9wi-EGhpvlb6w$wBtR?2nc9*K7Sv_4Fr%xz7tR?;X zV-i`7gAIHPY=5{u=rws~OE!CdWN2Q2?s5w`IdL-2u6&Q%bMfT!Zu+=t)jR##!J8U1 z52PpkR4;rw!Dh2?%8Q5ihtQULd7of8x5TzLe>@KCo4aeGr!^=#s+v%xS+0T(VGz@_ zlX7_yQv{LZpo}p(iFb$CX{$n2dEqIT6724|T9?ngdNUs*NXxf|vxt38i2P&5glr*( zm)=#si%P6z_T&}|VAuJHp;p+vgN9>Z<>>;HWxXHgF!8JbeZXblN>K%M?z2O`(mF4~ zPtuvN?#ai!$}^fQdOz7#9hB#by2?gjR?uiu?%IO>ivWeX9A3w2lQKTu? zF>8&J4OaXSp{X(dDD|FLc0H3$GGBj3A|YPECyMj>hx$N6FmvPFDG!v{8jZ>2p|eK< z{=@oj-o4s3Z!wtl1C7r_0D@Horgq;try1Jr6}jetin8F_Sz5mU3G{_K9R;{~e9s#D zTT!CbdbDs3y{djIEGfA0^;I{I^RysZbh?g)*FKr1*J0F#eAn^gBvJ?l#(uf}~D_xcb|x zp#kw5^5lVZI~)GgtY-OB3uS^L410xav1nEvwxF|Zb1*TsVu!StO?@q!h6M;kV=p!C zQukd1iCn~25;LhtdvCajm7|^UOgAk%`3ZdQHAYaSMbG=DwwoODzh#KpN#G*$Y>d}4 zUR1xkuC^YF9a4Y)P@z$igr^0cjoUWb#y#+OeS%^}W3+8Sp-x)B6eHHbnE}l}cx#?xfK)*Q{t$97a?(VrbWmkP2p7Lru4W>DwXWybqW+z} zOmmLi2cKT{nV)|Sdd&0h%Xwad(s5GHZ85Oh)9PpDn9n~P*rrcZMnIbtDDxIi_^QF% zXrZN<6-VXj9*>DhSRWwxy)s6RP7;H`pVlffj@JDbbWiOjPJ=YCNFsb+ z)254f`vwML5_eCh?~D*X=!`UPx83}G;-FD<3|lO1py(}uAh9hqGw>S{r*3W&AK>|k ztwq{_GZ5B{?~exOES9+(>>TKRacvxE*7(vp;`xHWqBx9ah6)oVM)6{mWHfEzvgNbT zu4jmp_g&wl2Suq#|7Cte_>=L0LHcn|v z)Eu!fUydR-m9aUR=pt!`T)s+l=1+`Sw~fn_&8-PymaN#4bjA( zv3w*fc*nlPz3ME%BKS)G;->ahr4-~>SW+Q-OsY3eAN!UGg*2iJ7bpp1h1CqYLRcIl zuT8+=TgikIUW#w}(tbeTz-T|TWmM;5-G#`%E^7(HfeGfNd}NepT)n60#t7j2P`Wyf zQxAg?H#mmrlP8dF90WbT3nYtcA-MI!qWwz?)Y8rj$c|f)k~e1`Yh};WV`|?m-5+WO zAkUgU*gg9e==rwVEgeF0;v*hlcfC%1+J_#^wE~IB#XgtB@f{nA zA%}G120ZYJC0Kxv9O=H0ernnmE7jpqPkOAR>daOR!`d~(S)(r=;+vq>Yk8x?n^tGub3E?r$u2Gs(-dN z@s7|5TT%^IWZw$ua56!R$=ub8EsoCY=F`Rcj9L!Db^rlk*ab>5hkjuAkH1>m?&s) z_3vXG8J$oA7SKr3_Gbd?rJl;;7XT`pLiTExzaK*tN%O&LeTi$U{mFFTynfi1!5z?Z zjVgLbw@iZXVNjdWuFD8X8w&e zR$O^s7VJy%{&yReU~3wB(8cl;*Qa>HQex+9$S-ta^AA-X;LnUJ49OS3$7dZF2 z7sO%p>9f+WHa@K@U-JzU8DKFUt8RBHh>=hneq4dPhA_8K4wd05)PPZApBDY-L+Q;4`MTogo79vw zZ^?}?xDyWKmjmJ}b_f_dv>t~a)aT;u7tEffAfG2n1D&-8tt2i)jt)!C?U%5{;Rp91 zzclwZFM})ZoA6Pp7D)`rz3?M~q)V(PZtX<9#wq-Yj`2C78W{L=^d<0jlBGnsMPuz! zLtmiV>W76fg4}2e&MttZg5H7xNK|F&m)`w>sP_R(>nvy_8)?3ao$~ME0to(B3xGX& zSw(n-;^<~_@X}+yyw!T%!RAk!h4cRnwu7^?J3xxB?3+6r#kq+z9xv1u&^!c#_ieDV zT>7ntcN~qtmuY&yMWg>l>qJa4=twO8-DFYOxEVe7ZLm$FK?U1oABmGi3Iqun2ZxL+ z7nE7=_rJj*Uue^8tDq@@Nu=waQ3}&VnWPHk`*px$v{XQTFRQ(=GRPf&Zzv=q)ZxMp z53Z!u+{Y|$QY}_ay?NK+IXks4Um?_H^{VhR+|2kQc!5CJDv!AU8!z3#}I%mbmY+2;8&ggysv51c+PLcS6F6%h;Eg>whplD-uR~hoYJZlfm?DU&- z>SP}K>tgDq;CM8ZNm`V2g!UOns4A}aSAH~-lGvC=G)W-nCD+N_^tR#|Wca)2{Ml9{ zII`m5kSsCrqLQ)Jv7Q+~hI(=Wnh@O@;er5|@rl;dojTIS<06fboo~#aiQu==Z)ev2}fX0ueUQeP8jstYuTYQc^ z;UBoqpk8WD!Ip6>``N!!*vGVtlUV=fwLU?;*7#HjR!rtUA9%*iIIr$N*w`WOLhDT3 zRmszOGNJzBO^2%6dwk#UKT$&tV662t1Wyc0H4kJj;F)ebX~Al*(u;cc((95(Jkns@ zTG_@yWy9sY>oAhM=j(gZbd=HCZ=;C9&)uH!6k>DFk^jn4O2g|wdKpB8Ulz8ZlSmtB z8;GN`9UnX88kHS4r9EQ^fXR#+bHA0Fb#%Y=b=IaJJ6!plKXS~}!Z6OE2IY@L zpTp_bjoaHknIt^KSyZq4t9B_4^a!_wi(a$;LF-+ZV>(v)xGME*oe&Zrqy_SZF-~Dx zVNCMDmDW%5p)L_2`4Z5gn6T>D(V$wCjoTJw5e0u`mTGcb)b7}G0E>sI-;gY~`6%6k zg!7H*bnAf;z3K7bEaZ|sFGh7Z37lki6Aa;hX03hh0?RwMsEGmc1 zNIWJcxmR}|gR<$kibM;Ie)%5+Ve%P+0Vc(W4+%ly>_dLlwt$(w3H4+ zx*1nX8h_=!FY9#i6{@#J2#w}>vyot5)N~QF)ndkf+_S*VArb3oxJ0MTq>n%g_;oky zb3}L&|V2E<0C<&amy~^;4askUV9Pi zZ>?jt^yfhX3n7+p>VU;@r=zR+Uwn>zS)dzjY56KS;!#odRi=6;*r3?xPDD_Z2Q8VkPqrsFu|Iny{11%2|as+kV*R-)|SS1QG9gw zU-xcicE6`2L=F8+{p#17R~R14P-*B_X4S9$v`mb>ThW4*jB}E;hPp(pVKi*UiV$Y{ zM~lKs&xuJiYjr;07*KKYa=K{<-v}H4AjWRPvIurde>7>qYB74KAxLpF0J!ZG`HC2vy zZI1?Jh-<)>Ma|om_|i0BuwDz~nMWE9B5Vm>D?2h0NKUy$3s66iSGpVuIT*Ly6F!?}xIZMf%m@U2|v_a(zh z3K_xfO?b;RT|*7Gc%l8F^!ymReh!z;sAEWT_Qlq~sTze@Qh{Gbi6 z%{%I5tmvYODaWXU0G{MHuGJVJ_;lRx?B!0W4jl!TL^y58?WI)$dNf?fGnP^U50j6@ z;%i6JXBg{zarp26wsq=W-b1rtw)N}uWVC1;tPejK31-D~=A9Ee9(l_)@w}-fT1fvX zA-1&r2hkCu&O}}(6@c-GV4FOAe~v_wB!DM6;fPcWlgZ7;j#(zUZS_?mvk(%->_oo0 zl+YJc6);YGwjbOqgGTbIot3+!;(gY#MAv`@AzP86ZqWdi=?6p*GL5Gs@)v7>$ChPD zC3EbE2hI~MA1_6SmaIO4e$Me%8iC<6bZSoCvls+Hcl)UR`}67byBhTC(05j+4 zCw$@Kz-&yVjTJDZGW%xhb2)7T=#KI$K7IHLouD+{UFq~1xetP`+u?K9nXxJW*f(U< ztCKA5TU(`CwQzW?Z{Ziz%?2XH6C_Ia$ZP9XmfL20byQThmfF_^nu*f|K|mfNe$q<1 zd2bJ!+5F;o)zg;d>a4#hw@$eipLR`2)W$S0z{-?ca=KCSGRarqgi@cV_N{1Kdt;FrL7KsC_&kTvb(3 zOhuo9Zo>2VSFcy=3sVk`>q{0BtQfYK8Ij+X*@X2 z=HqL;VR`#@G%!1#>hlM#s8Q9=ZryQyG_twb#HQKjqhDz8tO0Q_V~#=)$1U4H65ezb z&o>g=9aN>zL=}PjB9EX=Z!j+=(K=qy6oreEBGUF8@OW*IbDp+xy|Y?asCl&SMacg% z9V6dgx<=}+jz9QyFZ=Z(ww3K2$V>7wUh3-b!J{P*9i8U^8GIJBFD+j_YrX}Aa@{&6 zQpk@fonF97!*$k^2(QZ774EVk;`%GQs+v6x>EDVZxg^T6B_c(H&$<-pH%tKzr^Pl_ zp6wIT>8qji>g3(>~EcL^6 zu#Uun2dii)8wLLd<9balJOL6463AlI429FHgAPw(+}7abK?Bu|FBAca1V>nsIQ}b4 zpHH`J_d0XsLSGp1@UQ6PJ?tj^lz(`e<$Xwi5tQLrDhLC?<99;{VeaiQXxb!C@a69| z;txJgcwru}66i0i^rT-5=`dwd+bi6lS%^L`ik6+{_8`_R@@qYG)U9;UdBxcnrd-={ z>F>nKGOC4O4rrb-KTQ#|@zQR>0u-d2kL3QvK0jAIz@Wx)$Bd&0#Kl7wQMBlZ@sZgQ zi7gd9yXSb4rzy3cFE>*5EQCRQl@IExokNK5MTIyFt1YUx(M&|haWBo!UYIzg+&aF= z*;=%|;cpF&tMkxvs+w7s>7uK~bLYdWGnJ;b;KM&AL}I!5t@@xV?)=bHb9r*GMEAS! zijaDW?96`ho={yo$M}>CT+F#^oUSf0Ro@0liXa&K^THBGv;tL4b1t5t6N?$LRN;8P z?vb90y{Gu4n!&|9Ma@+z)tgvy^Nrp<)8cr=1R8vF@Q6*%t^U90970Q7hC-3E=Fw%j#Rkb674LetA8IH@5`ErXWj0m(F6Z=tFz)Jr<(p9M=dI>^QziPh5=-Z&db~PV0;eZ`7668l@5s^5}1-R)Ej~h1g-{)RPgD;Se!fwh?BC zpT4VG=O0qU*ZaXfp|}IV6N(nr7|hvIArlc`F84c)rds0$+jeMV`QyYNu`-Ed4zFZZA7^4wjm$qILMq`sp`FjE8N3wC64u5;+0V z`2N>$O;!IKZJW&Ib zPkzS-<8iZ*z0CSNyTqODQO{};f{i#nKGwR4YWab^;A`JB-@LbWc%g32-W_*`e!);z zC+yhTgEx693rkAuC+QQl*U_nC2(~k9+25`AFy4PnWDRJ>`;i|ig}v*4bCtl}J+-q9 z#nESjkW96`GslOpl;AA1c}DUGpNPm9JU!z6vle;EwU^NPlH?O*cX{+)TN|tM%T5!| zqGX?ReeZ0;F!-)BjegMxHc@IRaD#ihe@6M=N*RTWOLa z3Jp{NTL?;Ly#@c#iTjvPa_|j5dbDg16OcO2t-IJPm<1LOqWNTLhf|&$P#c1;9-ZlL<#LM$d2Z*T& z3B*E!h=gEHd(3Rww6IEMe1{C;moAprm%!3AIvDA}tL3@m)r1Yy%(bg4@vH`3H{}Mm z_>xVwoW)IGFn>xQc}ciM!qd{u8U6%3$xj#vtuqd($YC-B zwiWJ{X`d98)Ej@wxhkm=x4`u~-e}LJfPzJ#p){>S(Ytye02@_#rM?T4UPFe)zV}-iPkDK zqawY|2=c;>JD|pYA_CWslbM4M8)89zjmyXk{WfUZ=?Z^@7M)i5=_uoQ*bI=nwPiX} z{JylpgL)X$*}uLA4T870Vyyw1iLH%FP0G3v%B#D#kW@~7>gF#piT68gPa8US-?E?|kmL0}d7Qr+T0_06Xka`)_)|p5+)RXH5AXaX9~$%9 zIA$D%F_|qaZZ$m0k9bT3H*5u)c7`pNGWewe%i6(3?QvGuZc+mj=!BK;>&6}&SNK@U zK&c@74$lPIc}* zujljmnANAy@NZJ0A3qeb6GcC|e64=^o6Ry|mW#0=f~3!F3Ykaw?vrn~3<5Tx zi0XlJe=`YRpMn|X_lp_~Y<&MNa~ULUYS02WR@$%TQmRK4$Sai;>t4qW$r8DJ{MH;E zv(1Lih2oZ&!hC+({c_d+cKBUp8eQ^cY@`LjRBBaKfngB71Zxy(k=bc;5_SUx zp+Ew3xH(PqnK}2+ZPzgj30ydCPut_xT)JkvY{?V9J70w+nb_p`nUp+x%y)KM=-tzA zYo_yUk{@BUi}5;s))AFk!WFWY8f-a~5R>%W%^E$7%M$MQ<@O@JmlNGe@7@?`gGDK*?>X;**auapL5<<>2#3pVlo^N~t_r$%W}!4tG^Z>`tPt=OM9 zMTey<;+ghkr{SNc_@%AAg^}nVCaWW}MfuLk+-utS6WgKq;BV?u&Fl5cu@Ix)la|vB zI6>OZeMiIV(^$akND@qx7|URz$@!74Zk{hnWgny(tBgQfZ;PDH%rfX=d0M9SQM!)2 zOxxp?K}(cDuZfXE`^dlF1p{U1Z8)|Nv)Dp}79Se80sKm&h1oFxomI~R;c4(bBHBtX zSVvW)S!(Md{q`&S@^n3Elk;)2>I>qDge0RW-xtts?Jye| zQ?VIbFNTPVya{cQTO>kzj@@ZOZinmr^%}pi+dJ4*#oUl4qL;weBYTkhC#e}t`XTz3 zv6V6JM0hiEx#XF)BGzbB9}}KB><>)}94E;tHrFJSEh9GX6L@G)9b{)eNmO zsL2{rfclU7j6SYhjp&qDhy*NO-1|`o_FLx?$~P-%j24tlYGNs(fT9Ht;DC9U57Q5+WNwj!osA}3gHoG3C%%I=-T|z$R zMr)HHG%dQ9;2m7Q^2+Z6w8PZ2Rn1ES&FYpWq+ISpX8?;HqNGLTAC3HF9A!- z(Y9%gs%$VhJL)O!!T#0GJLf$YckQPuc5V}aC|uepsBCqE4w!9%tEER?z_SN8^}+E{ z{{N2`MQ8y9sQAc}XG4SX1fc1_5rC1-P>r&$X^TS8Br^EN(KC_&vMg+DFu@%o@Ytu= zO!TQxzK@URuq>7Y9_AUb9D)%<7KpJ4s z6a!V-y-|DFM$Tw&ety0`Netf)61(2+yL-_PsgO?dI?tRmhPm0%uEe)PLN=gah z%y&3;O8u9ceI^eCPOk*4K7ah~Ds%5q%dq*`y-)I4uKlwEJ#ao>cAcPR zDF`r;IXga{Ky3Q_iu25qGhJtpHw|4k-t`HW7F{sr+6adQXokysUsT+5k;)q z=FXyc$241eHfosz{&w|LbQWTzys;LoAUh4~n_wQ==#|cREQ`DM5hYTEtn2bZtc%UHnh|*v|1R+sZiEkK;`dC zi_Q_IRIZn6k1HhrFMJbxBGD#w`D`~^CjE_`QW|*BDW!cK{>%&Rl`39ieJjBA=BVvS zuFnPF$4>Vin&~iF0hN-Wmguc!`WDOU++7qNQpX&)2>-Y#I$}u+5_7#b=$vd01*yYO ztnV)Guaw=NKXjgqL{x8%S91?^G9V_tpSCnfekSZahn#rMa#U8$+xw_}b(w}!T?4-O zavi=kRy?nJB>kPl2QaOvsSzFKl)h3;a`l9#Na zREV&o;i2qfP$cCp835tz=!={g1zxx~d$!zRe_uQQp{!w76M)uDj1?Upi-m3Vyl~uq zHYN5uOgS{-FXP}lqE=)l|A?(dN%mv&kj2<5dW6H4#*bPkj0pF2;mrpsd6OG#P*7cY z)sKkHYl5%@<M`+pJT)7L(tyIDrid!oHq+@2P?fP}Otn1LPZa z+fafliL5J%GOK2N@-bsa!l}GZJxzN{VgxO4UGL0Q@VzL#aSk3W17qH8$`oifVk81r;dLjE9$$dq&s`d|3 zz+OO!^(Xw5Bb2}c5^t{DPFh>?#~(G^-<%~2Yz&Rd^hy@f4d`XDS8FTwUgXI1fq^N);Uw0pA1|_2?o4(ee=j6{l9}U2XdN#TD5;2tKDVdrpqag zm9WI3NU=Edwgh?;-sTlCu>c9OA!7e%`D`fMaD{>hWqqK54b^a)EK(}-Tcsa}0~MB! zSOLglwAy&YreJJ%pK7usUFmBlpF!i}?@#w9G~Ck}UAR}Yg&|`Y0^`}2aFk2qU-&nk>#|&a&rDD#NhtYA7ght76ZfKf=m>S z^Hl!)Y1nA=()j8?mRNOSxHz0`aD4xEey{Kij$*`TAM;I!mfKanUc2!6g!nHvAh?{u z+wK&~6rH=T2n z*;X8I0V5nIp()!}myr0SM=w58Q->O@MC+1zagjiyMZ3J%8KX~re8g)YNk(dD@1(uX zgte59H0=Efh~k1NoqbJjD3%p|a~6@iRK53h?@E>F%XW9p<)r-0uMhk6gez57zo=so zo!mB1)W*GfYQ~{=(V$7k*ej6kR_l8?(=U@Z2ymqXO20%^_$TiA_i}R1Bsj{O%S5r9 z3X>GFB=l{+qFJkN9_7Cf+Ueoh|vJ}I+L zZ%s=#oFmkR8pi#Eve~lSZj;gU#d>UW% z6fKj75{0F)6D*n-fY8gZZ!A4 zfIm}{pc%gkctcscp$dbJ5nB=f+8V#Vzw$@rXCZ;5zfM5v&)cPBo8+E@b%$|LwlG&G zTaAdVPN5b24^sZ7Q8PU;3TdP2g(KI(QyX zlz%@c6yaWbXYHoXG^zh;<}xjqMXs)~EBuyaT&qySGYu6X_T3x=GhynSXAD4(!O(fLB=AJ#c zGM^tadhPArCxl-h`*Zr;l9Fg*rnkfK50_~6qx^0Xqx}(cff>(iNo2bvUsPvmM23o~ z4s-3=c3Dl_TrzwmJl&-Ky4+jNT>tFSZvkzn{1^Rebzhl*L@}q-V^F^~?|;~J7R@Qk zu!Rx4;7A4Qz8jbS^B%0r=7CNyqoWtcbMg)9?`bdAz*0VDX55mN&c5mV38T}2b6&Ih z?M;o26sEw%_KTaJ?DZkBhvH|S&}L)5g4P{*{A^#Zq?q&XS7Bck+fcRp#)O^lRCUFZ zP63~4t>;smbw!o0k3s3qo0_NoD~AHIQ!;Fa_@wbcD{udhJq6I&C2+#@Nk&|(VkAp+ zGc=TfL84au36%`#*2%?X14fnDdn$@NTQ@!Ecb@il{A!29CSX8licYvOm>(&3z z`iyCTgG6Y)x~0N{ZYJx-4Z0J4AmjQ{W0ZP|>pJ6LXJtP^#&STkak`gr(JH0g^G&E> zm|HIzV4N&urCfqAra72!$zxK1;d-z1hRWUJt@TO7-?0M$(BK)&I!HBJBVr9eZ(7?7 z4LcA+kg`;%5Z5FXFfM{KLjN>YR|pWt->dd*-2zKHY)}Y%ii58 zh}dZ4eiVbmYI0mA_2p#*cVBij1kJ7-wx{buagFmc$8S3oGP7xUi+c^-N#3qY204x8 z^TN`1`#Ss0j{%@x3ZGl1I#U9}{$VZAq@lj7*wd7J-PT6!XN1#EZe+O12i4y`QIs;3 zJw;i&O z_a-)eH0=7c3KN|q_$*^8AExfczWl*@xWuGfeVbGFO!^Jb+E6e=04a~68c^U^gR-X_7uRrCA= z+u7oau9y{(hAOI)QA47sg&cXJVVlR#FOD)3X0A*} zGnRV&F=J{HLs5tiH@ze@!9af z06@jKIGA5=F17I+beO$MM#tTPQase<$C5-IQdgfO6R|!}XjHA<;niFPkg{mmY?1}( zMdW$8aoS0Wm=!Dq@jqCgN%y+@WHOIm-_sn1#in%r9R3>5OJ+iBi|-ZIb#bs*{Semi z@Q)k4zgi9X%@_l^qgRjFiD-ii`ocwBs(`z6mnisXR&e~2EDC4^!Qt9m8%J_B)LHc7 zn0WnuD?OG?cyqKffQ>e~EnLB?4+j}4H~#CJWFPQ9R-V%DH(_JkUK_^*`|YO6igH6& z*?L*ae|r&Tf4U1*I7tuL7}|Dr@f6kmRhQa(MgeV1I3S?IdgagppE)*Q;H6qt&SLrm9rAZOCG~ z>A}{p^5mLm68F$9O)M!kOqY4`Vfk*2Uip*vp%cu;H7f(#ezuRUNL!zHYe4E z6x*R%Qhv8pew84B6qAO|Xa{TeTdQ{X3{#kiNE*WZ; zir1RkTAJ6i}sS1$92RA8y}GatvG-VU={pqQq#I?ENpBU zdCi0Fx9WB>Evx6`5T|%PXc(+pX82s9n?a35px9J#L;z7v^sP&sqi{Ttf~+XQB$H-; zX#a&s!u1-4mACc7)D>h)56H16gIm#18X~}KJ-@$*c8iwgddMgU_ zJlnST!Yp(RhiYAu0i6)6AMJ16Ts7Wr2ZBNMvGoR!LgI`5JkFFN~h-7P+? z;f|vWp)jQ)$MEf@@7ZNS9M3{_DoShDB2QUnJBy|TL4Q;-b^Fu$lo~-wd&|FYCxv~$s%g?V>eo&d_GAgxv8F~)VU^-Nv89#opr$RlEIzO-# zdBglaUkc5aBnHf&Re_YfSb{9@bLJ8eTA%4xFuVf^1Pw?ExLG`^l-IVV)cESC^qis+ zMu&%kDM~Xf?l_cQ@9W82Cnz&aU^seLAh?>7#*+K*m$#lC(zuH=z?9w3Ywz0YcneTmEFKEfK;S= zsL+`#G4A*AcR5pmLQ#sPPoJ59^b`VMhC1P{B}$P~xn)Y{4de7eL{HM;h7Cii-DQ&= zBBJjvtOax5*Jty^%T|JoT4M8~D^?G{M2s$V1@W5$7k7$X{s^SQ^uYOVoKIE2$l=m< zEUlro3ACjNQOlV1XS3*pH5CPt-xrdWvQktUDD|$IedBr&3}EFA_wPQoSsdggoPI$1 z*Y4zwO*o+f*ax~qHl-WE0IHpfO%;byHj?fw81&#H$ImWDHGIE6=n>ctJ_;mHqRw?X zs!P7pmqW}zWntweyH>V3E4SYJ&5K4t<+FEUwKG)Osa_}9EJP@3sgC=Y`OK}4cM{VF zcXa>Pxf)3IB<(Xqr33QovI|Zx$bQ~i^RccIe|yL6Pv7!~VFJyK4q@(0qlfwpOuHq| zR_*L=&>eTbV5J;hFfzSw$uDLR5a2@*!10WowuY+7)x`xF_ru5NSA`N4{qM2 z{_KM?PhY)+oAZ0qPubXhY`mp8K}THX&t?TOq93{R^YcY!$X=o-_EeMNJ{{>5^8gIqMKP3wC#8}JH2xdY`k-g9Z%F;tN^MbVe6=c%k8r7e|=o| z-Q=?Tn@fXq+Xb0Lv56>BmEkjOvi~8|wIJbO(*0Qu@wPd<;@;7dAR^5mudm99trRfs zAT@es)ngJTx%o8#38K`Td*9{bMOmPA_P5IIbtThYL6Clj?0NRJmjbs}j&c=_t( z6mxE$+&TJTII*X}DSkgQ<=m%NcB7|kMJ)Z;^guwn&thvTlFc9A212IiGKjOJL{Jmn zGYwBgosd#6J<-?6*PFQzMi9V=u=+Ujh^}!dv0e{*e}6@Z=gQ?X%8~ycX#J!xFb;~2 ze+N|{aS%G`ik`U38x3IjV^`x(*BQG6((+hsKB|mCUqz%QGLW{p3wf7?CsYdzbfKxQ zG(r^uu2)x>92=%IfZ(-7rPz84%hAyvl&Ykn!|k3hKv{b02ngGa1CX^?a_=|FFGz$) z1Fy*mL|q9n-A+X|YrTXJ@F)y){*CSrfdZYadKuZpMf>|B^8Xq4>7? z9x^qzr}1=M#nc%2OG{~X->%S@6+o`u0{W?A`{$NOfFSj5iWY!${j2D1+97@ruV^Dn z3e`-G%mQSEpGT`7H?aov(T|~e;I$pLPFp1DGP$A)nNq^h?CF7QB)9))j1|^@P@hdt zNIvRA8I`Y|nopu@l^d(J>j z##Fsl=E*!n2pcU>{}OKzahW;%^?#SidlCZ#&Uc@Z>66fQG`?+39kd6lgiDW`Pq!g8 z*pNT1`Niz-ggs9z6L7RpSWFyklnUw}-waaQ6x0K2UOi=yID9wst$#1F`IdNPA&YnV zmL;RuY!!qYHA!BCpzz~}qM`^G_~eYwiDn4Ts+9l5XxN9>0a+>k)E1Ik)iu&{O_!qi zrsRLQHBt10i;*C)RgLm+!*?~E+^Ff`Na!<_S50uEGU}tP7l^l_6>Zr_&R2LtuY$ay z)}&AY2VVZrCUi*Ktb#E&3A^0-O!W zK}DVTkytxw>CN!-cmAw@MlR!nlToX?MiHeCxXg+sS2&j>|LVPux(V-yin)UJNcOH8 z^VzYp5OHgXXz{pG&r{*8|Lu7}62F<8R<0TU$r9!-zt~J&yRJh)d1}KGibz_;|G7kB z3jzmW=+fsF2XId}a$oSNuzd$hF~Jj{$cS;hN`Xd-OQu&e&@aWjeP}&oV4#$pje!iw zA8k4$J>s5fa<)n#i|Q4DD2Eq$w?!5o?J_Jz)&8?ttG?#U;5vg za%n6VF?-y2&-`mPH6K~C>GC~`8~Qb2d`seND3WLRn10_SxIiLp5%L|$`%m^NL@}<- z77786X_a0JMYps8cS>Z&^HTZs=q5OZ*wWIVzd!8Pf+Y-fFt=%%R_RI}kCzj#_3-{n zYq2&Ft%5+t;u+I=2XO`=>?iTKt=c3kQ-%NIIkeNy%;V{eC--vr-Z4q3*aRcI@mGxY zm+KmL#7@?2CbA^gR^j$BLmyAqY&K%G)a_yh{+tVf){=9nSC^=)CI6mt@Sj>?)Fq zf;*KvTY2Z2uRdu+#Qk0Y=0Fmw48!0XUZ7YFM#4_`AFu1a{sQm5D0iv3`;N0ZQgqZ7 z>nAksHg#ZAxxbzF`|~3Z^7Q`yAkRYKz93-k+^d7VOH^rtm#LX7@O#lInV({iqiP6m zaAm0a-xhrrJWqhj-GmahYm)SiOtrzvA&KK#Pxxo+)55&&JZqB-GTA*&M$~ODG*;ao zzIH{R#>X-t{u<0QQ<<#CXVC4NyTo9j{-XELbCz9hK*nvCJYsi6m6*MF)N;KtnI(o| z)}MDMz1(d?JYI{M8XI`lt>hHuwR!m=?9sGhhAV#sltb^0m!s*h##B}#Ov%ihjvclX zOkVz2YZLN9Z1_|5PwLKJBOFzI>Dfd#nUSCTHJmEvHg(iJ7WT2hCEHK1zuen`JBP1) zXtTDK_$`(eEV})-?;`=db)>@)*+bi+MYrERBp_|t`ZIzXh*@4M_PbuBtg{K-m(H%` z3aR=rpQ&2MRa0X-8_2jxu|p}K`IlWq$>DVnXCD=x*d*#nIQnL1Yz_Pyqj!l_J_4s# zH_1bHeug(j`_vx0-iKljiW>%W+>>DgU#EQe!0@rD)QTUkF7#E|Nn{tusLj)|2FZZ% z>iHX2D{|{eE@HNJ3Z||s9SRT~fPl6~)kQGh=~h%gzS^d}@uT6qfMvo-Le-bv!~W}L zMLJ79KMZ+4Zf+R-l`_m)awTkPV20N#6;GMH&%b&m`W$OleojRtsdZFyN$Q+vo0Ao8k?;psqT$8zKobUga9F=lo%LoMVLhQE66P-C zuazD^d|En%QznuHppcMVj_FS*9H}j-@9usD88v{NJ1nUfa;NakH(HN!1j8)QM}qx4 zz0n))HM(bSYUNBjvmDKf_Y!&}B_yao%?8HJ{rQhiib#dD@Tno--qZUMHddK%B|z%P zZGZMe4O7v2~wHumO7PL`5-wl;Y$0XE->N&`IA(!J` z%llXzbBIIxWG5r`C0~1lPs3Myaxc1iVbx@`OALSFbvvdcc#F9V{_4Sd6o;}ylSC|={QX)F2}3*S6J zFSfnP)=IGkiI#=lkDC{{Yt>Vy8y?Gdtc(B|NAR9ay<9JjpN;O791<_j-`Ja6ns=|; z$mv?FCRRN?!lA_KH<6W#)_W`qWCzZ%xcl6VQ5-gPx5wWcR%=5#z=TVCnV`|9Wj$?= zv*#&2cY4HFP7>}on~V3f7f>Ba8+$K*CY5QEZ2>X-q$oGqH|I&t2{)*j{^yMCd%^2Z~g)G%{E#&1d4fl+ddgFll`OdRs&ImQZhhbV^gzbpVj7a}%sSsD!XaTqbf z=hx~5WWCId45_lY3+rGCLx7$ROJ78vKDm^2V+6_4-OQFk+k6pp5*$^de(%PTsS)s{ zFKjPlRq72FOvhpos0PylCcv)dO|U(*x%kAhFatbmzGRfnKC;LKr_~QF8zCT?ROZLn zYK`3h*sqZ0nvzsvulVE?YzJ7YKiez%W?xiSk>OtmYQ1OT$N#b!*Z zP|qIH=XDO3han~!8k!2~jy|}pr_lIk*1TZ&&jD+EA@t)t>^`RjAVl1A?;|Es07wd1 zATeOaFgKle*7;%+LfWv(C{n%DktdFA% zCKvn(#kfsV_0vqlOIM5JNdg+N0s_TQM*eyz{7*j`Fnw`_V~(eUU;hIF>i^zjQVeXuO{u+0qYVv$J3?0zxLMUMU7?xoSoTr^ zT52(Kr;rmO9Nk!Kvc|N${D=?H+v^_&BQW_BuTuvwB;A=NJ^=}-9Cd@=e?&{+D1Uwq z_4zGzG}Mp4W;*&z_0=yiv*I4R_K0j;?d9fW-^Hvy&M&%}8kKJL zL;(Kd&swN9f+%>O`Br?%$v4O-*4@UCrJ~+h9~JA~|9e=XHud05t(!5qjJ4X*GqG9% zvT}@5>*hl(5tj3rPKMhWcN5B(@xj#wG&m$NZgZ;qrf)C#6rVkZ{DHb*-eJ4zq(Mu# zL9a||O{B%YIZbf;dA#H$NqcYAg}Zv=YgM ztMu=>nyS}y+wxtm{6j|+d9KV?nl=wr?>Ae<&u|DcU!*XCGLQ-ixlYc&z`aY(Hb=|@ zT&+`tqV07%p$aoa9Td%cwo6gEu$`DkwaRyUwpu~G5qcV%;fjTDaLvdYlS_MU&Cqfq zr9&Wh6F&B7F0h(O^o=s|XcS)Z%8V8^3g-?AL{jj5J`B7H&~1KZ4{mTv9J&K$nNXq8 zX%cDBQ#W!~B62+a78|WWb{iI@QzW2rk9uzTedUV+ZTqy-(ehO^G_IBv*K#uMDthM= z!;xU2*B+82;xTFkwF?0{LGO`kX{{ya{bxqBT- zZt_qc8~Q?0JfYi=`wUU?=Ry(kM{Upf?}H0_;5OZG2QX*C_O1VP=(k5Eg;c=L#~y7J z$NwHnz)3>($>qG%ogfh+5Ke!(%OvJHZF6(~IcamACI$AlKtG8Nz17;=ApXkje zL1n0E_*ZgF?r09y#3=FQZ#)OLSf141Z$eBs1z>XK^Ao zEV6B^IP&Qq`=Ei*h2H$#@!W;X$Kd&W%?uaRh;}k_t!q$h5n{XJ9AdQ*Xq$rwzAu#nVHkK*V8Y)l-4R zS43Amars)(NJ#)41VG4bp9gb?LXfFLW6@)(}=+YdIKwqXm76RaUwKAH7;l{Tc*#;qN>9=PYHO=lt+VlmN9l72& zGtn1f6QrNKmtGvi5Vrmf&^?BAq7l(AwYX}?Xt_qto@8fAE^`H<(#YjpM~<;J6{Aq3 zLrkjkebQT9;#YxYA21Unw#j8c#>=UAaU7l!;=OU|qp-?wQe13m0M-$M#CmT(3t(Bs z%EAZJ=?lDZz5dU|WAo?Hsq^-Oi)h;I*rA+%(UwXOZMk{EO)Pr!G)^2}7KD1wWVK~k zckuW1#C*Jo@yK&w@e|Mg>l`&UvhW^xh@R2dl>Fn+@W5f`)>GSeARFkArKZ`uBj%LT zl_2eNem=Qg2$tQm>=Yu{jEC5pmCEp7G}4%y)cqu#q!Ye?qu(FL)5F=GOi3x1z3Jcg zTltc`d#%14^oT!aTGw1C6X$0i4GBrB59`!C2~$-+E3YI@w!i*wbRGyuS^NrgFmgu*Opo=)VpKf7ObJz~}Rxd!F9b)RTF;B*joV=Q9A z(+}2*plG^oAqwg)UI#FJ-vVxMpJ~hXUArbOT-q3pRWl67h|TGvHKL6nJkUQarlTFc39f-B24ThlhTo^S$ul9h4PXG=oBiOvWrZ`sHNqf5^i zCFUq)#i5qvy%PkjVlcO1*w`ZI5)rUW8w=!qRSj6`uxy(w66Y3;TDT6=;Xg{sRwc5q zl{eXTapUfnC?>MSy`9?rFpU0-ig~AL;;VOk2#)>qZr{K{>0M5EAr3t=s^jDy;q-Tj z17O|gQB^#Sdpo4znBnqniDI<%1`+z*F7bWn-@DhYb4U@r#E{B%j?x!sTR@th4vEB1 zhu1-2N(m7J1ps|3@FBTml@cl)?@#u9lEND?M-0ld*k1KbHiTDA9ZM~WUiqg_b>$*J zsiDEFWb3ie#S9Y~Xn(2v<)0PD^ct9dho5~d$u=)t>^|Fcw*5SZ%s__3#Cx0 z_N+s&vt)@{d=7a!U1;%? z{=W`ZTSls{;Qti_tMp%7r=kA1*#eDLr<@gBfhu~YYI9$gTxHWB+kwsKo^8v%5s{TH;dtx2rpLj^9mxysGd7uA;!}Z|UDCE)_)fq4JEF$FK=AoiYDtWKZxExg6x8IUVRe_p zj4UFAul#kLC7p)P6awgz;!hc++xqi&`%l={KxnU6xBBj7Dy^c(KvekYi8W#TW@xr{ z(~E#l7#U_V|EJj~x!Xs>545$mDmOj5{q{8w**kt`(<$Te)DF@`IMD3O=s66=pZqS@ z+4&2;QKiL(b6@kVmIpl6+ff1I>$N*S%jqm0FBFRS=Iou!X8gHh`>z+`Ph*vR+Gn8S z-aPmPib=DvyxyLZ`p!AI%q!p-Jq+>+B%lG96Vau|NrZ6OXc&&vJK-&=OTr|_lB3dO z&$Wna2C*3qV7s~5Qaf^3DX0F>?&eDIwD{_z=kRZ@@^LZ#;&nm}aY4N`CJPK-!e3B( z1=Gb+Cf71&j+$Pn>v%0o)<0mFD1iy-t2(kF-hAg6$cQGzWoU$*luHp&*2G_mZMiBw zcWB0j-i9S&^iye8;F$i`O~i7Km};X7DXB{w-I@}Dt}UzwSzq!4gn}xAz0#%=uU0*X zUE?fkvwAv6^w^H)3zr+cLlXn*hoU?qY>qD3nwyiCw6OBG{g7wYIOOr~n$dES7jPGb zKrH}SZUtHQta6**ozUu2-g-IqK20ER59(OUa36;Pkj?M*Y*W~Vhmkjcl9^%e^h*#L zOo(uNkA*kk`1N+n(dpUoX>sLC-Y+|w&=lgYukNJ<)r|11h})KJzFe&C@faQRDBKx( z&0Ev4^?g=y>QFsJb=2UmBJj)LctS1DfetA3>>>APT|c+Wm3cZlvXSjmo1Ht>r*bj$ za_dEuWb{8Z|ErW7D(Wpt4E~unyfgcQr^s#Y^q@b=|4|}aAZkY>07U8MKT!H(&VDdc z_*@3lAnj z%SmBCPKh6;8m#^*K<&`%rH$Mxm}!YV)bOrcO+Ie3)^dHU@!$97j%`{w(6Mc`@z=Gt z7@;Wcr3y%6fGzyBkVd&)?|hRp3Pn>a+c^p{`d8TCeo7h5Iw&?>qz+G=tE)YW=e zrVo0@^N={%BC=m4Zz5c9RqBJeA}S-E-E*D&848QLRtvta<_z#_-D53r(C0J|@_{8+3E0(6vGb&U_z z?~4!$ocy{5nM_tLbhm$cjTP?R{EfW^L~9>CXdX8ipNn(yioeYZTKhJ)4x@}03q2iW zN2fn}NROs`oo8G2tIZLj5(dZJ+|Ex2;cr*xFL0X;uoa{KLA%jI*lcRhne|)?0P#!86^D?zX3|2RW_B|@t~pu` zkdg$tNsA<0?rQRzCJC@O!SWC9HUU4Igu>o3O}V?f*`2AQNbWtmcI_%V3K_2@=r#iz zsw`~Z14rAc3gpG8!26MDVk~)S+gQ36V^T7zYO0WSY3Z#1st>}r^tRmZwtG!n@Js3x zS_^-c)!*J*FVakF;ofdHG+xYaAYl;Akme78w^m`IL;zV@E9Ger1^wtf3XXJfvx1tt zbG)R#IB_j1>mTHq6t6#)wej**Ic%ijCsk*_Mp5r8uJ(|rFxJMXRX>+aY_$)?>&TIk z@cMG|Ah-lsN9EHZW70j;lcyy(pI2A#h7#NB8bT0qp?mlIYq6Twy&h2FuCtezGIw`M zynbS6$wzFDIf;JuB#ID@c>8(O(rM&aa}bVgK5mgSmQ5ocm5*dy5f_>~+mtF|2Rbeg zU(kHDTD<02c&s;-g)-#d(>rnG0k4h^VPhTK4l;J zVmZUOd}tLAPPy*M5PdKf-tdb+hhfA6Gz1#aWXWyeS7+@NcxHiKELIp5 ze4Mm3S{xqEN&@|(z2I;@6R}_8)bh8mh2`{e#@&EB%mB8-Ball)sTx~K#OinN#bjBH zvd#SE2YR=%lW9n>f9gp=X+%sef1eNv-RR;^8Up@-!>0agRk8lk zCkSI{VVcSFW#$&?$t&WaQBU9!0Bt4;e3YJ;+f%WjNpAIhJVJ*v2G7M~hy>?&*>CBw zZ=E6Sifu^lWIofn4f^F95l2LZICjq%Th*rx64xn?+pjF72P5mtyy~XEn;pu)6a}}p z`;=HCIzD59x+E;V?%-aR4KcTGC?3ki6J zz3G$=Xp1d)a&wS`>q1SF`#9M|?ZfiUPWpQvf3O!r=J|!vX5OnChiZ(M{z%&*!gNyQ zw44Xx)eGNrYfcfFAYfg0t|?}eZnattaw?MdKd!M!&vxX#JpOKY_P1Uzz)WIlD9STM z(&n7?*Rgdq*bD$t`;Mbu?L_o{U7(;~(5?Lxh5c2m;PORTz|_`accPXm*r&1aFPilK zIu;bDE~J}qtp7o$R!>s8DiY@OCb?Ohcr)r2R=#bG+Kav;y*Xkv4w%3IU&XvTcKjK^ z5RpF&TH*nW6+;wQ{+|}YLloj7mLG=Fb*rPNM@$LB?NmVk;*UCzX|yO;skUct#i&b~ zUkj)h7FoCamC7}M~f_+;2OHfffUekku$J!VnILGt;RcQIuO=Ve^kQ0=saIY}L7?nQ} zj2L*Sq|zg?t0p1X$9_tbnSdgFr`>kaelX;E>vLMeZ)eWIfbNh@yYuu`<=d{Avng$t3Q!e7Huj}{_u zA(|l0@0rcAti7{+3nPO+-J`%BxX*#@X5(;aJftZ8z~CQ$F>fUYxpo-I%4HnLUKo47 zHY|1vEiljhW$o~F?AmiyFkQ%TO+4!Hs(iI|rL7NtiO}dBxnB z>2{We>G3}fJTZsO6nt)AkzGC}b`jLtIK<)ltMAW(sr2ZzfqSQ$Y*tY{Sv9K~5}AcF z|1Y!T*u>!gbejDL_gKtuJF^Z&H8x)mF<>=vmVLp(_lru7i%_#;b-f-hc<xRap&aUJ7QQv9tY#-js=jDT9 zm`LzPF*Gq-!fo~H*jyuIGaRV9tUVGn=?X(j9(`KC(J!?T*?Z=%s+cJ z0&jK|j-YbI-gXM;@F2{bP)WvfaHv1MUcY{0HqB#{5U!${x@`9Ax7$5Q42CjRAlnOP zi!T402;oSJc@wN5J>^i{RqsD3;e9$us9(GY6s+HVU-akgqX)4w7jp7v1~1(s(*nI= zXud9r9gyq#jJNpoGx^(*luz~Vvu!OcI~e+OGoMuE$h{w_+e0KneSV&tRC-_W<|3T+ zBHR%SCKv+s;>KRhppDtOVG)j>HRX8RJLn{bzen%6xKmVB`B*?E6m|S_yyNsNGkD?= zCU^BK_#S=hG_-13cFp5-$ZJH<)p-Bgk(Rss9IxPO^H@c*0T`&p##Auy*6i4HhEI390(NjuS6NVcFD{Z3%QC9p$JotOM{XOeUQ za+MF-bNCd#+O$L$A5bLZW${EhaM z6T4p+^nzQH(4J5f;^!|TbzU)uXWy8q=5{xZe3~rn?VN#KaGGYOG5|AG5}%zX&PW^) zVoiKA4ap!OLNfs`R^IC=rYYo!G+vtqNc)pXWP^CYzms{4;{IhzbBC-k7!K>IhfwYl z&j`Jg>fG!4N>*?^T*lsiy@-{pmdH1{A#{hG>%4C4$V`Rb0)G2_me@4h%+y-F>;F*o-SKSy-}gjRP$aZ!C$<)~XOY-Z zyUS{AHENHxR>Y=75^8Hvvn^`VL5tdZ)+$wd)+Xln<^6tt9>2%$Z~pK|uKPUqoOAE_ zeCkS+`S23(|K}-f^?m$x-aF+>f_`w{@$l)!@-iol90j3EbdFPE4ncjp=f(}h3#Jhg zfu;8!qFxBTc=3gc`QDJaUAr806eNTa%0sKOFBfTB|~-n0L4WZ1LfuUcKZy#uOQZ0)%Hnw+tBw)?kb!m7RmTK$-7RloM= z$`mmD6TbD45`%kjZvsNpun8~aPqFAk9gzlKV~!u>GS*}MVn3EAGsYMl$?m+}Ot-#~J}&C1s*$zAJ5~_* z1jP9%imolhf%dlV0Li;#KD6Q;jHB;v$CU@k)vl>c^hPjBC=o(%5ga7xZ)`csvm@CG zj=0qLv{JH5?sF7fM)EWJ@J#|3M#|NI1fz{&x7DF3*M&R!-kkNtFMU~zVW@uhq^Lvs zPsF{2xWBl^FxFUy7y+DQpe?mi$qOnKZdBiAu@KUAS}CR~VMH$CwOTafSrko|r^)%f zd{uU)G0qS&AnM&&?vE4{As(I>yo-hQ`@QfnG)s$4;hjJz4KX&o;K46AY@yjq;6>p_ zRu8v&`2^PKW+E>|tt7ZDV#~kFA#-*W zDbCj#`li_bI&vcWPEs1{yrUyJGmq?_|8;}+#=5?3$Kj@qS~u%j&j_pE?zH2M|9(Su zrAY}MOeUmng4Ld5J}w{Lg8cHXwyI)4b0hX?Rb zJnU?54R0~Y+Y`!Q&CvX(h}^*)mxAAOqw0NE4701Mq;v)^EeE@^+b_|qRVk}v!QKjW zN?6dFZ0Be+9u~shDu#Hs50BBO!p2_KS0$&IHSV4@x>wTbr0AJBV6!MVA1LEoM)c~V zc_leV)o@ItDr0_Y3hl!hxVE^B<^iaKCbgPN8V*{i%oFny+E{y6>p5jlEVyxCz$IEskV{EeZ^o z_u8olS)t7;=bQ(k?RT=}MYRV&R@Re~Ij8+&*DZv{|6;bUHKwI+;L}ouPVed~UX`An zWdEtR!WHw@iyx&ynr3vJ5_~(H#J8eYqAQ8!B$p@_c-yA-H03oJ5L8QvuOH0ApB$bo zM24R}@Uyod#U(O=uBu`(M7C}AU{)_b32s#N0C{<^VQRPY`;R}~911jhu+!9r(_d-G z_OM-l<>1(ctNZu~Df9W)&omBK1?$#wE@bs1AL8ck=r0QkqEiGXP9#tx^`bz2kbe?~ zn-2BFB5?@_tfU5F()F!I!r-cy-wB?84wt@J-}*(BAKAici9qs*5tJq*jFax!!XI4s zPQrmV;XA9L{G19hxOOGK8d~`C&nL65>RhO{P##1S3xO}JrO;Cf=rm2>qp23&ebUlxYLC`lSgqADSCd9kxGmC{4+ z35*Mlf^=t$!(Vie_|LXNA$b5TJD-DH5 zgU5CP7bB~rCkQJEEEmuy>{;FFBdeFf)hR$%5}7PzdDHFWXa_ zlV;R3SAc+ByM3petEOHEIa2y;**)Ahh|D))GTTi(%v_d0H(D+pK5)2HcaEf%ME1bN zh+Du?_Sl9TawoMn#@PjdnS#!pOsubFFUU_6%QY-HmyJLSotM zieQ*OZrZLf;v_r$42n6KqyBVubIeUc)~-eV)9mf&i#!D@V0k9t&rawfKTvGI?b(5u zfHUHv%jMOQFB`cyyCRBGFj1@_?OF*@Y(#KnT@Dn7<^Bzs<2JYtZkzb{jiuhZvBBz- zT{2ClNX{Yt&1-Wvs4L&F$IAT!|LpOB&^ZKu--7ZEICNM^! zNbHWo{wy9 z(p055P<(ZZW1GViXU+$oDU=14@hm$f;6dek9Vr2<#JD=YLFU6W9rBHhRb^|95%t z#7S;usjs&4hhospv-ZU7o#03>fzM+f*sgGR{0oejxWVsVkoc}v88t5ArjSB4liOD0 zFhqBEESI8t$Qz$9S1=^xxr^VK#Ex-?HaLoS-sd3(QX2?j2&uTqSNn4?#4*UWc~uUi zwv6X>Y&y__A6&oVg`Glt6aNw{s9BW1kiNin+a6su zXvC0_1b-V}3ATihsxaE{(lmx&;*7sDp#h$Qfw~5_QzT$g2K@9?An;uqQQF)m_P%(I z2u>3x#+~~y%>k9DkTy=6;z~t7)_dmbY9KXoHSBKT&3a{n>IPrG;uVcLy2nYeVy+O* z`!%|)c$l=8kKOF=!H#!DqU#Im$XMQZ(?(S<^p5pt7#r!^=TAFUBJ0B{Mp`9NLm!)~ zl89h*K>!x}cZv%&@?9h$F|}tUXUL4)qFWIEBw1HJpvE^^3X&7 zu=Q(mH6tE=!13*ganL7Ta@4RM_Un0jX85gz6X9n)=AY+htoChTP7DoTSpi41$UBC?@{>6 z+~ppGND9`yc9M3G(jhLorug#>hpVFBpEjI?tFQRO?ZX|puQ%@Rgi?-o`YNpHy8o7o zzBYH8bMudBoh^AHlb?COb)my@$%*pp`CJ>-Sv>}Oj4$Q(Y+!3-3wyP8(^ehOzGN@D zm?$6Q9y_V$KH)ghE?<{A<-3!kt2BMCiJOd+)KEGcy0N}V-gv58H_$bd2wsGOAQP~@ zpV?MfAyOupoC~a6f|UhI;ONv)m4G<4i62YjcOwG=c|hQ9y1o1bn{;%$Zd8e1+oA9y ze@S(x-FwzN5M8Y_3&8|qg?A~vG>(zGmD>1{Kt>!7-dn553!f6&tj(KO9_H{ zkyKC7B2XU5s}n0Dx`EH0CA(kKFikLzB`;=>bm%&dWXr1HJI4IhwtC&o-wPfA^4=Ns z)x^WVldggO|vCnd1*! z$Op_@hj1W|R$^A-?F&4dE>Qd6t4W2Cpeu`0&9Q?z0sfm%Qp4StuSTV+gFI-c<9C)U zIGcty<^?_H{5ef`Fn!~zTW6J5Ya%`K@PSlYG58t*#XyC73f_09P4t8G&k*)suMJJsIY`< zUlBoG8P5ju=79u`{8AoHnMQ6Y`!iJp0>PGhbztHq=@$r8=Xxq*1e#{{vUkirg_3g2 zNonl@cQO6awKeUn2iiQSdp(fwB1eI+J?G1hf}e9qJg1KEDA=vUT~8{yna9Zm0zds; zOXL31WUSY@^LFkvFP)NFC()vZ7Ae*79WObg_c9-ani4pqG{OptqM|tF%~*a!w!yk= z%9yW%w6bZ0@oIj#R-&MEo2Mlj5p$0L;jTv(CnF4lKwa=E8E}HJ;c=HPtFrDgcZ*l+ z4cxw_{3S2fjW6n+_>vLZG94G~FS!+}w0-JW%Ms^uxe9ZbW=#|Vx zFFH87zq6Epa(z*9Os$0J3qx&fhX7ZE?;il$$q(FIlC%;Zy>PHDz-OaCNO2coI!s^5M^l3>5Z2^6tUQ`n z^mo1T+%dkTl0?+6F3IRx)G>k{5!m>bC5k@mly6<;`tzc6UewVxClaCLpy0{|%U!di z_hxh9f3#~#xC&_(6SLDDbH8WqzjK`ZHkCH@N4y2JQgNFS&q6sG+9lY}Xo|pd=ELH7 zQ3$Y3t$vVqF{!2W4W~fkI~&}ThhypbuQI-8Dt{=^I`eAgCJp&anPQ-J*KXRV$ARtx zNggV5^U{Kust-cAZ^}|fYG4y`c=5~*#a1dJauDj`<9=%KC@fQO4*RN&DMITmIAxe` zUBW&zy=*g?7dcQnI`J^kEx*e<<{K@9eW)zwlTb@z*ie!bdvEAN)<&6#+5$ZbB$u-e znc9zpeMQrBj@b4A1j>6BrzHWwlPL*+Y+ptXlgKS*cr|`nd~@qMouWx!fJ_8LmQ*~# z((f+A3lx2u?iErHv1~Q*Ane*=&6ol(aS_Ho_#~x@FMa($Ao19?>AU-OPGW}=KW=i) z5uziX=(kk26)63Kk{dO3cz2gbz$)+9w8D;Pw0t@(GfQf|6iX>iV!D2$#0gf zAI3{J5?DXNO5|-%rddRIkR<^>rDqCtN0`>H+^(JiuC?0Dwc37#GAj%51B2p^L-T@azlVM=!w*%9GE7f80Ynsi|$B78Xe|40VzuAfnoNReI^!R_Ihf zv81OAttatJpw0qkE**o??{M@|3xAa7CNJlD2s*1UAejn{=Ln2Kg`iXDu5)GIa!Jt$ ztJ0r+|B>ju-`2qM+Lnx|wAteTcbA9q^e*waP__~a;wb0c=);WAenC8~^d~gRKrd&F z8}*$f6JZjcop0`w_u8809+z_8cHfY{dJHNpYn=drb=^%xLffFll=A}_qhue7U7nsq zXP~_-OLBStaM52SoFMefUsr|!1ugqGDv}`NZOwF0#V`@tHxSd$7Yu=&{Yq&Vh|q{i z{l0=C%e}?+d?hGsuR0|$LTcl3>@jwjWzVjalOQ~Q1#QC`bo*Bg~ADeAV*B{6 z`>#Ag68`nPmiAXW&LjLM>wgp^6G`xUF4R=Op2bWK#X27il)zCuo>ma{AoaVG3xQdP z0F((wVFHvr2(kAK1kN8Xf1d-y{z-5**wmszMgmc!vMd%A{^sUbyZE{ghp;(?k}c8s2|s zz?w=+?8QQ|>)+PN>KG|0X*eL$p50NEMmch!vPLb@E8Gex_GwSXKO8(ngHbwocMFX8 zTlkf{(N~f;C!Xk>%y6p(4TYW<==jsn7sm><2e%7WqUHq6_3a0?p`6w{uUxCQ$Qs9g z(>7;hd=MB%IBO*RmZe+s0;8YCy}#sl(wY_3f=8Z7ehYXDDsMznORQg|gmN2x7dFe3 z{-eQCtSatkKK6M?Wy@)YaGkHYSL_CnUn~r#Cc_7UvnnaEXWZU`rk7z4#XE5{3 z+xX({vSYz(=^jPj9foLNqGO1zK9EX+@B~jjXSl%G9W%sljMWpHK$DSk{aM;lIg?6S z#0Q#7Y5hn`*BvNU(`D4Hz_(UAIsVB8_lg2|8ZVdY-A7}v8lR_>y3Fu2ou^xRNBc_16=-`I4=HFlI3G-W(6!QBJHW$$ zWAy;f-LR$igFw~hkMK~o2H)NF?0p`E){l`ePU*{uc}XiDykC`dQu;mm7~tv9wDpMq zk0uF*E??Q|dnc_H(bkbnOkGNJe_fAI`Mj4z9oLs*5id|`G|2cTKn+#+O$`gxH&b%N z!wa^|VdNT5vhw3;UX^+0DTk;Hga!_g0HqhX<1mNPaD*d}l{4bJ3yunSE-qA;^5+qX z#+HZgc|c+1MOSTKJT=ZO7cZh26&z)=*2#>Cc&$|7#T%!O51{`6!3(7tPLD!Yja}Zi zlrIQ9>Q+;*wg6h>Ej3wHY_9YeD``kl+)4#49tfx;1(ABTIa*Mz>I0#gN2?zrEzY)+ zXF}m82!ZE?2mlzr%W5tve|J#|^ayQjOexPW3R`*bH1gnV=7^2hf6oklX5RwS!@OF4 zR{oU|T9@VBN7qV0{CN)bgVz^+Tu-dITs09?^tU#|X4%gy4h&I}-E0=)$i|=4p7pIa zTK#?KqVPFwNZJBOuP*J8U)^PGq7kVI*FO7PXfckWh%A;rDsR0`Y8RpLG_RV2CNaKREoU zT!JxVn1T}p{mhNmPftUQ#wonPgEwC+>w_=@Yx=O|RTsB!`KW0HuL_a3QlC~Qg7W{Tbvdf15*abMz)z^5Dp!4Sw%N8`O&T?nfDL`cKwWLQ+O;|q!-D3p)-|vn1Js5P3N4b85;J@`ga;EJ~ zSZMX@tWIg#nE?=RRiQTsyghr^`jHaX9W<7Inds5{Kr_-MsaRp+z)@E)@GlrmxLRVp zO(r?;&qI??ZG7wG5L^60Xk~zx9n;h_lN{tL^%I>J53iZV_>ThCjWzWe$uWzllfwql zcHBLCxMOgD=d9zhyRm;8PLsozXAK~nn{FWAgue;c){-Do2t3-|TFajs-|*YrDqYe< zmU-y<9&lzmeyeKX(sIxVm4&jV8PJyK0`4e?@lwTklB?nO$cVuZAP!`7>;p-$bbwFCDVLAe^~L~!tC z8$H4y3WfR20Iq_Zc%!gcE|!5}dOB`mj4MwxEt3u3cw6hFzco*Idyh@j_eRdPgK1Q+ z0mmmUl%=gutY6BnI~6B(?Q{_6kcIf+mgoS8V$qS|%mMqGe4w!$m|tR@MuQ?gXkF0&zF%&aGCd)j2$UO@OWv5q=64hn@vMsp37Y%!Qsik6 zygDTDcm|A~FLBB16{4JL2_*?W8C8G$(cEIvN>l|iDHyrAK9VSXQ@#glK~@eh$+ed~~aFPT`zJ zG*Pavug?HUoi~}cZSoQyP<<$V;5)9UTCD!F=^Nck%*kCr>G{ndYTe^5j`tl=+tls2 z5ea>|y>wT}x6)rr|Hs{wR{rxAk-`vn)%+?YWnL};-{K*da9v2pLK%yCA$cQ?58}_3 zxdIOLilX^Sxjwgl_oOWUBloLcn zpL=d@qEttFmx4nJS<37Ofb6ygOK$g7f#W4@Q*GEY*E_z+BG8_S;i-D1< z6;E=V2l?Qa{E(}^#&zMJVKKaq+&T#)a)9D66MxzgckyG)-W&EtW!55b&Q(#u>yxvC zZ*&rmyOuODedjV~UlEhrPMWR)6%U*xyW^~TdZ;@~1|G6vKF?Bp`W&hbT%2ksosGLi z`@mLZ8&K$J{GXcn4BzL^{dQ+HcPCr3@y~gEV}#UvTa(J5xqr+U^cq0uWFM#6d#wYK zZfGO-|DLB9)O{Gjt&Q^N_+VP=hh_pAuL0wT-6o@#LUqCX5+so!kd3W z|3D_LE2lZyq8=gwlLIn&4#`)t7VsHq+RTlAs%~Hhy}TlTHztHk7b=mUetgta^x#<% zN$}-t#Y&CMtDt52Q=h^&(~0-VTc6x76^3T`>5W->5sABuZ(UfD2o% zp17Iu{7-b(fGRonZ!nr%#5G)B80bjx7aeK6!@nt`T2Ga3dv;^?DW#yEBb!uOA{vXr zu<=JO>fel7u*f)J+FLe^zuK8j;&_GBvwigU(TYv>LI2U$rNojOj$#hL4n0w=AnA@n zW840!jTA9BLfHaYz)+3>a};nnV&@MSZQ)D&jA!OmiL>lGT5y+Vt{R{pY-@K4B~Wny#A! zi)q0>4K3Ka8%qUTPS??Ey--P16swBsr1Z3`8@DDssr4MiYgO4ONhqh~99=Fry|i&U z^7B0fdJa4ye}=SLBQA>M9ijAv^!-*@@_k05^cyw{RSqIB)*sDRwlv8%{&FfJZej)L z?`uzWNl0^RR>}a`4Mm_p+&#kntLRSt$*ryu=-_wUcA1{NeStd#;A%@Z zH4$JS<=W2CWlck92Fgr3DH#FhD1~je+1> zG+eep(8MdnT$EEiW8w&f+>VpdulM3Tto z`Ods{)`BOeHnowyhnc{lfiE=vsAIWzn~Wz^{$&UAmhvO_2;idFDTB;8$8DKn*6q7l zg>lRbqNQPz+wg<=I+c;KWrG( z27M|+pw4y|LPU=N88i)^u9wN^ihR15H*kXPGolA5JC9u(qi}o1de_0wliJ6tvyJ)1 zH+gQaKB7E+&yS*N!=(+^&`FtPno?@tT9Ny=krebk6bZ1;6xL9Dc7rh<{_U3MgHiim ziAnFf?-X@u8vut6-TJeIUq#4|!(VPPq|zX#?fe8DAKjQQ2+U?gi&+38ygqO#RV$^= z54eNdbg`u|oCbGXm;W1x5UM|6x#cXk-_}=qJV1Q&@cB$o_{QQ{S{hw3p!k<=M*Rng z=Qs^~z-^SrJ@I{CH1bPw$+?O{uZ#Oj-9JN_nx%EKG3K_Fd%lR<+0mGnX}?C!g4n|& z=#v&JZ$am@Q{pBSGf9FCMenEurs$Xxhk{8sc^HhfnnlJs? zg5?i$5t3Dt2i3Kkz_XjXNYTPT5uf~C{FS&+j1BhCK#S&5Y?Km6H9YM8Ye)e{A_Ik` zzeH@_w#Ey{sv71u_)%g-N@&6O&}Y$0+yXXmN_=CK9&@@DQ9CpPlpKBH0)13hpCgh2R0b`x=RbMIbe5v1$&I-4B zPgb+>zD2!}Q3-HZ$G-qM#$|+Vdg{rJVJ5?|#Q~~zk0|50CrCK?&Ab3CS5N|e9Gv3p z9AEmiPbb9Y87i0SYZXoj#4~hj=z7rB9kas*i)-^Qu8ce?wG1Y}D;{@uB&HSte_`5_ z5@vV54bHHhFMp|De*D^# z?^(yU#ebtoP6+TbOiTyR<<9E;JSV=91BHrH^ zb&xdVzGhAT>;3X5QVmC(7p1{R5eUc#vuX}g=Ef2uyIdJqB*n6ohXdtl`(q>h*I=Vk zxs@w_xL_;|m$I!z*<%bL>iK4VR3FuqU;ytFukF3QCNF?UVH1nNk9j?le)7!dgEJ zF^s~Lu7o;bax4D)I_Oz0I1QHAX$T0)^;_%bP^PvmGq2U=99|?Zi-coj*D42>$XDNW z=lwzgQ7%^DS06wGOP0R5PMi|(E+d5|o>uO}WDY<3msB@V}%h!e;I7SYoMhgKZO%TiqL^K43OUnM)2ERROzhHCTRbCI z1GJrq^?sKj=Lh@Q`^v1?D3X@qt^7|DQ&Ch^g^s_QLap{t7*XLfp&s+V&XfBZaXMUO zS0)<2zG_J36*|%}Z5FeouzfO>#abrsxSw^MYl=&TG2eQW-cS_8qf!1O%R|g#A;-I; zT-y1$>mrkvu|w}~5ng)s-;AAn7S~glUJ_xg_u}mE%^&(oC?fUbGXJV2sI3df%aw<} zoM?2+Jk&@jTX&6d7@EmV(4&fEe`+kd+o!7eE5b;~;zHG)uKKV~;18a3pae{FiXT6V z2=(o~A|ZOTmVF7c+cgtS%AUOV2{-(U_=#PA`lXi%a2wyJuO6G(>0*bZB0rPW%UD|` zBakDL*N2>aI-(}t4N?6=d>9VWL;>e#>28xqQT7z9$&i5Qj{P!?++tVjoMunFvXYXtkt$MLA_vWMK` zSV1_kJBX~4N@VhK`n^sG2eLN!%5`q31%8(nh>9;QQ*XemF_f$DmbTt?C6w! zbjUEu2$e$XMx{Fka?@R@k>*5IW>;C&+C@{hU6n*3RQxK-T6L~AprH!2{GhO-u~=z` zw|`}O@~&m_RyS)|-z+j*HTk$lJJ2HLWg<^kXdmccPB}5exNJ#fsBGSWlWm=ifk5N# z%5`kxtsJ>t7pnSyXQZTSQ76_{o6!D}iI6V#h_v;7ROf)TZ49a2-`qbT&J^_q@;iTN za%Nq0P*OT4_C~tY{E}w#Y~##%x8ZYxgp?n3!xHwvn+h1fSwOaL+vt+=Gbr&jpT`@; za2AQ&$RxKE`jVtE!Jj9EYa)-TV2IS@4eC^WOorJI3qR7+I2Zj7{l6W(J`Jm{Zfy&g zSm(_&cy{WgmsT%HCWt%vof>N2G6N0>`Z~gqnJ!?_71?UAo<+XR;Yi5sd#gFDUCmn0 zsP$fk<9L66zfb5T9Y>N{)@=1H)d?RNV~x(^IJ($d&Na5#!FPa(48pAcpH?Iw_Kz!T zJu$gy#y^^bprMg772^2HfUPm7`2;JO$}Dse^84~HNz^MP_5Fr)9!X7qggup@HdZZi zi9n(i;=Li}l}3}_OF~K?h*cZlqWWRO#)VoSyuGYMPo|-;Bt-bmmhDURYn!4$*O|Ui z)l97$loC?7Zb{qOKY;okcT|RkJP$RMH((Qna`U;z>#Ou0T2}q=%n{$F{?R>En{W> zDtiXlEmY9$f><=hc4AO$Iq>Wr!j0>(PSma+U^Cs2@^pq*M%+WMGA^<&w%Oj1pAD&P zYkgC`ESp2RnEw5S1{Q1oUKJB_0ONupQt~V&InUvyHMEX|N^M@-jg#=1y_#v-#el6y ztC5?{udj-aWH$n_j@O<}`7UO^@WuRpqq4?rtipE3Wa;0kgGI>;^Y173iZC;qVxXyu z7+p}kQLQDAbK{dFUGnW&ztlR@Z+795Qqx8Ppm|-a4&hQ;bCWqUKQe;i{=X&ij$Qh) z#jn7G8Qpt%+wHi=iVTy_596*D5XI;#?uMtanh8xn>&3jrtY%X!HdzO_KBOP;nf?>F!oGtci5wIJ$v2$TY0u#;CXbm)v-&2%vxMr>Fu=U7jD;DzI4g;IC?XkNv0^jAc=g&!84rx~GpRVNuYN7fdjfG?1doC3iK85ri|y z_67TUg-=&X>6oI7V{BFoZYCE^8k^E{=Pw!?=tsQ8bWiTZkZ9H4rFW#qaiaWm&2r6FEx~AezRKu~#)Hh_YxYS|5WMII6^CLuzn>D7eWpicxSFnIV|cv%>S z+3=FL(M_$n0OYa}k=h^*jOHYkS*vPa3_uwtZ5E?caYodnS>bb41mN5E;QTsK`CE*&?1eXUZ%j!=(>1Nx=WC~%}MLV>gINOrxy z=9qZK6uYmv61CsQ*G{KBXpXs+6OG(%<6bz2euHzo;O^TBstK>QI2ji&!2F);D_!sU zL%wsD6M6H*b;)<8IfaV2wS^1&?DMlnWZvU{&rO!?g)@E5#}Ug`*85aLDT~BFWFMr9 zzs?1in;c#-csCbhlZH?VrM9GjNVz189fB>XsE^+*^pb^;F@N4q8M=+R&h^PKM|=e# z_*?TXxOMdb$4uP-1EnOs-C{}M2t#2t-^!0II@DpX>mMJCd~mAN+~JiLc@?usa@VvO83mzjW@xaj z=tnTwfK_b< z;6=B{bG^B~ra>sTbY ze%$RdZ{@Um4$ZR_ifLAs>M^Iy>{~=JS*16t{~A0E;vx60k#4Va&WxVch*?R0FyvSE zOJK{Ed3hLR77DxCsgvrI#&JjyOp+RJPeom@B&*UIN#5u;D1ZgS+JjT`1X2oo2&6Yc zEIBaBp2W3?YRLR`R`N}#Kp0P@zQ1zag0In7^_HAdx9F@ zp3}Gu#I-Zt39P!&$o=OkCCvvLrW@MOVaWme_b?Cgd+z8Tlt1qNzUJRZrya$YGDDZ@gJHt zD(Irf*jpB1nrtwo&0xB)zytQWu_7z)S@jMGfL}Ts$^r^cp=i*jnTuwWQs>m#l>?fi zc>J;O)>L-J#U{zY$=UX1@4oK^^qma^pzrYo^sNc|eERK{K&c+)GL?RR*T3bn_IirB zXWDM|?G9HN{v_pwu6x_o?ZFev5+IiNwy{gqa1<>wm99nAlJEWnoBKhs)qaqZRp8F= z=4%$01&it*$gUG=npWAD--(4qONWOuUhet%`b3}{O|G^6txm4;d_^WnesAa?G9uNC z@m1R`As(cyn9aC}eeHrKhZ@I!h6e%Ni)B2y$KO)z{$zeo?hOKcazoo;Qdc~OwYWcmMa*77|<&f}1X)8wdH;yUSw^ekg={j^?KmKlGTe1rNlh zzppTHnyMo^!qOMA4%=3>EEL{exx+kO5L@Yy!OM9E*&#K+ooJ==-qKM;YiC^~07P*W ziWnawUAK3`Cu-bD%Oo&20zG7k#TqnjLSi;ysk!^{1ocz4U$h`bZ+^o5tLKueTv(gG$Yq%m|lmrfCSc#Eb z4kPLJ^ZuYbc!m#3F4;)y8#7~Nyy-;k_DvD_Rq>g5$`?y3V<8v(-;>R>(qr>u`&QY* zjbDE;&t!p^xF#SdSn;`XZ}E)_#`pbvNntx*@T9>%kJhp2U{kE-nk?rW1vZ(Qbla&l z#C^!dXrT>vpT(@HO5A@w4Ssd6ih0ELMk=sJxJwi1hh_Sry`H8{L1za9ST{!}Ek9{#@(> z%5h#|aP_hc7pOOH*~EwINN`bU-0kc2%JaW}ovH5XBY7peM&BNKkB>KnxILP*->2H1 zO|XynTs0zKK6RJ0OMPN0 z$Djdj%JUt!aV9S=B$eBe>|@mltRm;I)HfJYhmx%aHSGOwd+2+p{LV!~GR^T#rEKoFB}fDANGD0X zL4!ZPL1nu@oKRVKC+OuXbwR>1-H883LI_YEhcDiy&J{&r)`TmJaP{&(%Wou}IJmjV zbmR6$%sxUx-nKTk;qt@+_ZHIK#wENJ{dWE)PTEPLP(*<$TKg!4HXFqs>==JGOaiS=3aDU00w8j?Bbd0D=JE}z?s3=ke* z0?+PSSt4AbNQ1L)o5x9`pC#{)JJa0b>mk3xACHof+Iw$dlmAg3=&Cj2jxwNJL7}7< zhky3ph(3GMWzl}n8lwJEJu2$eK-SF&lGM6P8n04Zqs+%kf~&s6;Q&uJTs4&_Il+Tx zjRGV_RBd9TEgru6Xyu=lqA@7GusPX|^Io_&T~B{O$R%wk=>44VuU7oA=F zo|vu@iN|wu$27Q$iwmj6HDJN=7!TZ`>JTBWHe!^Jg_URbh)XGQ@^E%M9JtM-GWs~( z<;8x`n@SW%+m4t@_JtL*px(x5G|T6ZMNpkp1Rd}@m;!*VfTGm4)zkoWIG=Y@IsWTf z%~cFnlp1n-@LRtCds66_N^k1%d?O7W+OXpL-^9clabcs1{99Ap{s|Dy#J<3wu@2ZF z^>Yruko1;povsZEeEkr1z>miTY&Us_{LwSPYp@Qt>J)-aA1F88_ z{^GPMv{56qjg|^gc=O|V3(DX}Q@vy-0sF{0Fc6J!!w5i^>;&iHOU;kAxtTec`LA;!}r9KL|)_v-ad_{*>q~JpVMz^BPi<4e0f8lW`u?wm$ z;zdCs8K40e#gKfOC(3jcX{EJJt(IJIjn_SBhveyCzfLM4*?|9htsS!M*&~-Yx)0L2 z>6Z(?_R=e76;a|NMht4EpB-_20;Z_5v+T!%+{cMqp$#zfKb2HpIl(tmamzI&{dd5x z%-Kza3*5bfOr1gc3dc{8pw7hBX1F=Db^2!DuHz{+pU zT*W4FF_X^Bt(!WcnJh5#pp`g-+nJLsLuAo-uG}(h{ULXBdNA!a*IMa#Ht~RD2OsGo zbKDy-GUtmK5fy@tF=`#3ow0Y7s$FlvO(gZ9(wL|V* zCEXtXa83mj*lXBj6OU&P_8$q}$(cIANTL%+xN^0?5ZXRYWdw4QG~@xKUWH}urS)xz zaRA~u2Vzh>e{s8^@+bn?4AOrP+@2T;S>{&end(h?X?nR?5(WJ+_~0xF9qmn<`ialk z{GDPOt%}B-r`(qf?E6kj+K;r7vE~m$$h{_ovlV%QW8Xx9cP(EXl}ZvU?kBy-D_ z$EeElNyN&0Q?Mm_#G7EOfrZc=Ewwx8^MJ^N5V3%_jwL{e)cRfw3{mE@`qJiw`13s# z(~-2~DmUJWSso6l+u*Q7BM?d1^{noOh>x@q$V-mdG~F}-&x|=1J_1_-dj0AlsC(*~ zOCw=cRHPy2INC%PqtvIB8bf($(jeQixW$_e)Lv8OObeY9Z6V|@ZRxe8%kLFvJRYeR z0ome60mloSf@r|r2MZd&!4k!ORKRTj2H@}u>>n+=U|@ORDm@}k4MSr4?qZEDvU7&` zRaf0|$!KIc)6jblAIH24$fdNQ zCJ{8KF?8Dh*BLRC$0_PQ7B{ratE!J@z|dNOVT9=Nw4%D5XyVpkG|o_bw%TBE-2cUE zyY}TN?eSZO(p6q7RZ&0Y>s!hEs}v2E@xeI1nj6TU4M~xVKN^x>uNEhfLDRD$6V=|L zlNg-7YWPH3s70c8+9<8WTVTv2jqBe4r=A-5ly*xJu1J~3CViwlDgKL4#j>G9Snvh% z)!Q02LvKnEN-gZSuPJ8ea$CKoVfLsb5UE6r?h@!_Hzg=ZCUV#d#HQ=YBIG;ma2$NakA)wK}ON0lW~oa92yJSy^B02j&EWCCIv^qTX{$o4s^FOG089`B6Fm zyuyx+l6e$$rqQv7bZhz=Glkltj-df2g;655BE2#FS z_zX%bQY;Ic+4Rw#}_(nw}|M0sXqqYajCbL1`^94r5(rrXs@U4406|<;cg97?=@($2$?D=8+7ZY4w=nql} zwAx{}oj+Yl^r%=F`b*nVn%aC91a1>trW7ke3B*Ijn3{x1=#zE{XN4=O<(8ieWne}I z`9Nl@gJwE8zMvewbwe+MB@85_&Voc8#CIeR^!ua^;A~CfQ{*oF57Qs@H_p&U(o3aSC=R=goEyWEO2@p|SF@a@6%YCWQU{S;mT*9&nbg%eP=}g|OAFo3 z`jhozwFMgt5oD-i2~ZD}!}#3x1YkB;Esf7bJV5xClQEb8$dN&TZ#LA)#=@U^N>+=0 zieA_*TLrfNxAa$;I1JN<;hP&cOP>+_X5NvEuQIXYi$w`4=Iw91!0HxnEFP|>th@D| zAq-UeT~yTkDz6A7bxYUD&ow+Er-=hX8JZ3Wp-$MRoLVB_e%rn}#Q*L_yk+Vm&<0Ca zp%U44VdWl;#rdyi|!s<{FXK|a{H98b-DcUf%`*jiE`2)o$iL}Ad zS~mJ)CJT#TK|D3RFH8n=#K6@h96^~F2yRC9ZWhn*LYubPo=0sWw5hjSSbgqeMt244 z`rQ{+^sE4jX}`L|4wIZ)^S1*+s0gs#P{2SB5+%VXn_!Wunv9NyUl$xr#z`xiRL1 z&zwBG#^B)d0Ogu6JzWrMGDDeXK1Uu0#6LzXmkd4|mWk(kD9&sq(PheN%p{`xThV~% zkiF$hQ;a7_@bhZ-bALo)8=L*6*$gREgu$_lxo&Bb%gWkX#~Q{Q9yDO{x$)d>-mqih zyEs=B0}d-vFMaRjSZkpBgczvz?N0%dJQ7eT(2WPQdS~9BdxyH4z7qLmy_v4fWI}|E z&21d|{1Efy>tB*$12j$R7UnYpJmDjTq5%yr!^#%4&!v>xg11#b939|z>`8>wtfDBA zE;;~~g)Iuo_Nb>5JfdJ)uf0{7NBoSL29m|Ru+^q&m~?;MB^V#8>UgO{NBT`|C_pJfU>3f(U8}Rem zP&{dS_31)6O!{@v0&<|VfERVI2@czF#t(jmy4Y=$K)KvD^%#$cd~JdT$`0w?og&x4 zx?FJ9x6VGh2FzIHbZ=I$2%bm>XsEPo?*f5zn?%?SbuuSYIv0bb}mXSx~OxFWV@R3Q6$j}Iig@@LnZ0*eH6LN4VH z_;ZFS7)#p>$__AkzA_QYC^}~njS)g-A&At~A^~6uJQFL#{viQ}P!M!pdV@7A7A}#b z6?0Ap7^^>*&FtwbS=%~}+DcIL8j>q|=1nc-NUkVx4cm+;W=oq$TW(8`enJi>L0sHY znKM+YSe#vHlvA_XmwFh+Fdc8IDlNUetHY}qaKa9i;)|5(-{w$t@_*jnuLX?`4L5Fb zocxeLjfJ_NR-QZ>8}>1pcY)rBI9i;TI>eR)HCQGk9z~~)yMu)_#=)9bD|CH0rAZ5$_dddLn>S4Z>hV0mDeBq`ZdWbbfO{ zslMwmrAZ`-z3`7RutZ7lGDIcU zj^Nc*P`=qh1N-5$dB{Ra@s->a{J$DdI#{J1aGQXmPW(v=SC;SPyqEKq{uIpoWH^#C zW`ZWbHBiDs$L8KZEt0jzTB9#i?{@e$X`z|-j8oDc_pPmE!mPAzI%6(5%iD{{C>Sh0 zxH1SW@}o$&vGvgALwP1N5*y4IRgHDDz`+pG$&W>N^9_kEJ%J>7(5^^J;C+}MeGpjK zmi_s+3rN)| zg7;nhK>3zT%1Z)Dl}MAFy*xDpiZCp$PDDg*7xr>d0Jc>NxgSN2{o!Lo-?`4T0zaV# ziX%LK6vkr^(kSYGPbs_|g+O6@4~3qu@L~W?WifL_cIh^0baPRfX?OLS=09E(zbO{%~0LNb2+AvARjoBjN9TG z$Ng^$gMKbYa+kxYH9tIly;*~1NQdoJ-6VidePQ8%2wGE$`+Rs!C*!PgwEKvs#T@_w z8>ponv15UUf8Ce?DYx#IKH9v?vopLf{7vDwPT%VxmKZ=V#|?#8v04Bcv_qKV&+h3u zsE;42vFO!f&6yo#A6@2k8q02cgLt+Ny9V*BtVIJjE6}icfz-Dc3wUSWBKXIa=JDKm z=~%W~RgB+i`daz&8ZMU^3yq5+sdZ&Uv31b8tlWv%g}xeQ#d#&HewlXUl;`pHsda!= zY5})-+E0%8a=_J3%^pl|Vcjh)x;H zBAgp)gz3`lUuo`JN=}AmYjmo|LDv6GbKHix*Q-|d?qyv-w_8&7QKn_kmwWTYyoL~$ ztkH4W%V7YnmHhJOL6*!Al-&{IOK>eljCu?*bby6e7&N|SlL^tt59Ma{ISldZx8xa(A+KT)q9`Fw63 zccfH$hhGDcVn9`Gzg?r*^AhzAgSGmok8go7u+rE%USK@jh_BgNf;|CEK6KdNZ2qcRvF77z-8Wvl$KKFehn=hr z#UTPu2uz67z5f#5zIbD9GQ^`~E2uL|IBbQ^kNMU2M{RW)Vc6`QSJp=$D>J{+%`j~{ zc!A&T<~@>e_bel6h%S5~h;8?-t0-SXK4~!ceoArb{@$-Gi0mSG>!EpGn48{GFS*Xj>u<}?A3+*t#viatG`J57KAu{>l;>>s*_8} zvaKU0Gr%1>e|*I>I?uqG#WekbAVHdQ%hf|X>8#;w{33q zU*&@r6aew#1q-0Q_-EAiM_1&p=E#5kD8+l~1OZB2p8v`V{(ihC*|EQb4F2^)?MW6( zt>K4|+TUCE|91I2ZGfI%q^-WNv-clY`Y(5RhxNp_oHV)vSkM38mxox10@1dWmSt|q ze|gaVxV)0&6M{oB>QMDx^y=EAoE_{lAP#0M-8; z^#8u@|L;ow6FUE|edVv0?Ela9(yifjy576?b@jpSdV~Bofx5QkM~6;Wq+lS5Ea^5s zoOVV4)F3+^4(-=~%oR;h;8bGw-~cVtzTK1eq!a$BD_{msZJ%k|h(c`NdK!%BakSL3 zcGdCQ*8_mx*#h2{c#R8I6*YjnXK>r)0uxWpd;7LmF7EDOKubXQ{QNu^Q2*dglP)SQ zrmgIFyv;8wE7SLSoz7*hkjDlkCL$7}O+q0hC2h2==|WVusa&kAsE}Ju8+`rajL$&>6#Mn}R60P0!R9gEx?lZz_Nc%l>c>&c=Peg|*o0AN`N z7aZZSXyfVu(gGCwGqbayfaVB)Lqh|amRrH|kz8S6fr}Z{mK*1)*1P57oYJGVTn@2- zv$!*dQYC$T`yB&k=QAbAewJM8OTawCxC7_j%t5%N`#DoE;B7+b^AnyNx#Cmfhl}OK zMPea905cf4|NcWC6mUO5I+8>}&FTlL+g~1X(9~mB^Pc*zhC|5=kbQwwWjNpZQ~t(2 zBI{tERz3fRiyDJ2xI1(!8=Hnz4!^h?()C;I*be*<0*(>M_c9)1u(dV(6r^IV)PL zX|Bcr8^6H;7JaMRx?Z=zy6Q`FQ4eU80X z>&>WrMoC;IL^A-LQ2u(r;biNU+BVPO;=6`<L(qs;XqoP_Phbh$otpX541Y1aadxmOjdNOh11>709)r7PVcEjQ$GUX*#oB&j28zA5}9uH`A+Ea zt-9}g6}0-rWi(Hq!#a6~rxg1MCTu>}aBod@w(0KI7fndmcAcEy;Hs&5vEN@~02Pbv z%tH<@{k+dNSB#@5z~KzMqKm$l;+TTTlTi_Fm_A+4@HL6OT4?HlYVOVg(}g)?NhB;h zy7yrLr0dc6cih5GiQqvzCmsRL<2m5;qOz5j>gl2HCT%-S(Q0!nwwj;C?**{aB4c3sTy|0eh)_Z!etc91yttVNCmO=oq#O)Ajt) zwz#cLP?)Kw2XFJ)O$VSrdb{;8NJG!FH<%iEdR+MTJ%5xT^_q%+H>^&U?Ph z-%enCVK6EJU;{Pl;scsvPF{HQ+)dpLE^oCL?mlo(77F;~|F*0gQa8L_oA0 z(ksu4(cqY5(!09v0O1%k`*IB2zQR$+Es?AjWrDPdIj`wtB#beFC;T&n42y|mBG~R9 zVXyl$aO@knF4x+91wa@fdmy_kcey)}2jSmm!m}8*gYt@oJT1n^va^xsfzEY3fQFv= z1~4@uE(NbTGfUz#$lee`+}qEx8&jV|b&yDR%Sv*rwkGn#xeD0_f+M%$p3PsURQvQU zB(_LRf1y99IJpLVvFo8hSl}5gX>E0%xyOgq-)%J!OQ+L{!nSatRm8u6KV-|XMGrI- zeU|jLV8fux^Yt7Tyh1Cnm`@e0z>)tURk*f37gwk2MeczD?sUyMCc6>>r2UDVB#_8` zYEs4lP~@JL!2n8mr{;1sdI=lb#^1pgS}2gU1m$yed6w?Ry@2iR0rME7)1EwqtT%u`446}!#b;!u)77|?9wQ_`oBl|JDmgEXD-X>48@Pi*_&4C|+o zz&br#%wa~s7#bnZ^u2b-30juG0!RCQJ#4HO;`q)r56dMn(m?=B(8R=Aj5p@TxK0NY zPy@TBGxvzZ3smD%-3_ncOZM%gNGz7#+Xr9|H#H;d4}{5oC_3bI{WSyo3h|~*#se)4 zhHpKb3h@D*k-N!B?!7i#@XhT-;gb4t|NH02c^9LflhQERnfS9hha)VCw*Z;rb7uA| zzS!>%3l6tX&cE}WPJ<^^K7!Xc>2xJkmwk%>FI+>%F+`964@XDgXG1F!pu`E1u}>D~ zMSF3=5w94a19Ytfr=w43%WS8 z_xJAH4wDOrHqC^`Fhd}8k`fA>>S6PmlBFZG>wW14eZR&cb}*Nw5>QGUi#XsB<5@{o zI(?OL=FW}|+V1j`)zjvA6?160p|x1L~3p#PVjEw8U9;orHb0XvX86QX6KjL2=nXTfBMOtdip)Gbh_ z;VVp$)|S`Fd(>Gy+nJ?Z9F%6i5T66XGLgf3yIbh|_9vzVaELc?*mgWzww$j?0fur> z6eysD?o^Jm?N5dzt@x-^6e4})qedG9mJ1$OVA&qT(Tse!F3nOGuT0lw41?y;)?otw()S0AxG2^So~ zTgh!e|MKAqpEL{yENM6hq{SroimnKKI1s=imfC&v7MLCWf-1kqv)n7HKVB+c99K9K zUi!bkIwV3QFI3??V9hWQ6#v099wiPO_Kh13B{lFKX;Z3@M=TKGZOcYJrPFKTO?jRH z6sPNxFku_p{kOOkv?)vAe4B~P3eo2R1`rq+zVs(8fJc~aARZ?)X%ZeV7l~gD%;TB- z#1?zec#5q%$Y&bA>~8H3oUE92=%kyf1`$r!&4^`>+IQ#qO-6wvCfa=KPhr-?`%uwK z<)Gk?55M*gFHUAu73Z6&JBvH68 z;J`wC^!5rb8PvK%>pe>EgS-BVkei`7>C%ddeT^{d#1%#iubyTgeISVAR$Y{LZyWk)_cR7t+es2!Iy83h?tzML}OnDWicSFLszkz zF@{Z4HBD30oLnO%uhA1jmEt`~D};^vqQPQ)eoT=mY0D4b!(wnEQ@}7!`)Q00N7W7| z#JFE4lMnwI-N4$?5(QAIcdxSX+k6r;gjN`C-MpQ*s`zy9nw1>pruVjYix@mm68+FO zsO;0xnm}^DNhgdlWme1RLyqd71YKMJBC4R^+RLwWNy0Z;cso3yt5g}D8(WEaO`<_E z{p?w*c$V+yzc^xo$uS$$!FHI%4%t_HN+l@ukXVV#=olTC#M>( zRe1tLKwHoo)+r1KUqn3mlKaG!(=ba~`HN*|@-7d)oy~(+BUyaYL(72o-Wnk79}}Y+ zcl>;j#ZgYwES_poiPrwfyM8^KB?F4DE+Dj8m z;iA=nd+5EMLmQ<7%iF7r8`I|-fK->sPcM!MdnLmgCxO(PQLrru@#~##rPYmlJ`C4~ z2{~6=%nI)a zT@iAU>S&I{PG3Qe5u9KDDlKHr-sjob5xP6zs@!*)>=T^Cq7M{#{G4>>d^82S7-+S|ts@&;M)LT(PTSkR-5=qch9@a`%LCiXgc*N3YVSAw>(^T6 zibR*Xrxj{R60iC+ym6JVAiG)3a%EWe2*4D`>S`%QAb&Wx-iM{EU-i2)^*b*l*YXu? z{UE-$H(j!rxz=i!Lrz~!jO83Tl0|?D+wbuEl>B!>f&sNN(EKu;)o~sjVJOKm^87(8 zWlE4s!?Wt+ybUys*on*aue9RyVJ_|~ zo+TN42@VS@i~_rPL1-m|ewj_H1`l}$SOYw(U71i`y{)eizfos4rHMleVO4%`V~pRB z)uEg4wAUy+a`+)c@|lkFKvIUz9=aV_d|x>4@~fbO7IYW5r+=iH3F{Gob@*GzN-nxE zUe1ZGq;A+AcXcc9ADY;s6@Sr5Q{+E<`IXG+)AqvPD|O6Zw-~nBSpduyKM#l&bEV59 zbq-V=138qF6kH`5cPT2oEz~=4MYG>T{MOenw_|&+NvbG%)E=z=qScYrh?Oe8B?bqn z*K2n)JbXM{gn>lt$fI(Fo}>ze`JT-nH2RGA3^xRp%K`hy>}0>&9;>mF1Bs48#;m4E zln==P#0F~+TFY+E{uABbZ`Hv~j~T59mNM`q6~ybvTXUD{+0HDs%FY-cmN%iF+zkt! zR_2pQVn;Lr-?l{d>ZlRKI$dvJAiT1!Y1Hv~+$`ZvC!u*(g5g^lmj*vxa{;tikz1iH zfB)y!C!y>+k$PO>`NUe5{m6i>2V z$%f>Yl=L^C1Hs}_ba48%>KiuJ46_Qoa{6-1;VXzat{=H1jHv~DlAVtAMPjn$F7#ld z3yXjWVd}H$x7B7H{>pd`rhC(E(ZpRVaWVo+l!@OM88Y6QW<{ z$<8+};ZL=XZy=lgj1OzjF*0Yb8|~c5S9?sBPwYyCR&uqt#LGw86uyNz23!y z+XTu7K2r9Inybj}KcM7jHLpf{h5*>eN;L!7I}3QVWg}P>hz~L-xCYr33eWRulgh*} zD+E&xM7+5l6Mt`oCruDi1Cq`NfmIb)3ZR9hGz)L0NK^-y0%&79>}s0`1K1ND-6~+1 z{4j%zvnGd))-76c*ln2}3oYN6Xr};=AO}qNbJG7lpP90Y6xA zsm0)uVgp*0CQuB+YF?;n^ul3G5bik08_@W_>qQ3jMTs;`91z2CT6PoUv&R#*p52=LYXsyX zWHwO{D7<)et(*J4851HA@KYlUUt|i=F_b;iIrxq~STLv!w}qRvleFH3L&eAq z<8doF#?eC{&S?3x7aVgLuhYQ5had4fH(zu*;y!gr9j8c-hg-*4(JZ6um*XSc23|-3`Ov|P>Jsm zw~>xwns->JrRak&zX6?EYSXj3ljgtrJ*JdEzbB${UjhfLg7p~*27$!FuPDHv;^c7l zT-|VOF+v~6mHYva_=&MsF|^%E;~jer*IY8*Qm#;OqN7+(dTHXC@nD1ETi^ zm@_xVq>MR=0Ru(qh^z26pgoXi*6_Uo^s+`?j%AYMOniWwfG}7c-SoNdMjp!GLS6`m zQl5QZKmd&}u)`$|1rXa}0t!e6Em!ff>F-#YcF?VU;pY%r1f!pS9bZhFVYtl9w*EmS z2aZPuW$ba}GY8H$INIe&_6(D?p1z*A5_mQA@B@&XI4aQ;=02+JqoRgGpx_P0B<@gd z6-2**J55Ax$iE%$2pvS2RqACSN27Sj#8-68y{{?5iRQi{t}C3lDmD-%MnT;%$LXK3 zMibr*Ge6X9f?dKs97}QQK>-se6f%Rq5FJ9HTY~r{Y{wlFj%c(2*Til3r9vp>4;9Z< zemHRocC5Lvdyb+zub1*+#44oi{>C>z3ipWgMSr6`)Y`8OSGke}S-CGuG5V>lR?MeU zDKM^fb5%vf@V-(|(fLn(;Zo)&aDKu?mIYjJ7^)rp4^TTgp2u|`vwCe4WIA&gUeF}p zQ#2Wv)+DF{mYy$YBOzE^pN-Ylvj3j0PBVnKn7%KA|z zK|L@y1ltWIDi6#oih}({gjyRfGN3NG8_!Z}oe`O-ON}}HC!=U0y=)@2vPuL!t8Br+ zs{U{P4*!*dk+;V}ZkMOohk1OF)tYz zPQnCA!}sgm8`p_2K`0JwJ_hvO#dAu=N#=OY|5UrWH|ZELGkI^uf%ox+#?a@Do=V$a zq&#*ywF{7A0$WW*qy2(!1oxJn-#zJAjU(flZ+Sy9NF<46qMHIbkjT( z&#hqc6*~zAh1grs!%X01=HY{VPyO(mrKyhtpb}Z-jhpWL3yZ%Vnd~ZHi)0EPy}v$E zd0T7@>e2+mZ?@tJUK`~EnIM~a&J$$q59H8gEix0Fa-^%Zb!@s9d^yX_jf@{ELmq+UDc)5{ZD`%ydr+hkuY^J)fQp%Kq0Ao_2 zu0X&PIU((1W}PRoFbx zxCT%rXQAci{Np-i6Z*+d0(EU@h7+A0su$AH`g-Z|HW7oQL$xtocT`sh0E^gYC}`Wr<&4UDf8*ha(X|&{7vBjy-QHy86g{>abvndtE#+ekY$1 z1v?tPchvdxQB!R2GAXAK4+fPVeNB5Q-&mz;Fz5{k_PlOXdrdbH0UNxKUYoPpQr?ZK zY(|d=YV3#U;`=sS!i$>gDw%IzfbJo5D%9)yMhH=$ggwWwxgt>=5j5niYaHA~ya_=f zD=jxN$UuyT#SLnA*@q3}C?NS&rThB>I=Hn&1LgQaM_O`&711np4ZS}-|JO3io*)A`<3dBza|>%kBNQ_Omv}i+)+3T{0pN= zuJm+vL$M)ZF?rqfhBN5Y7>r6zewRW$EJBb(iU~ZjEF>z?IE{-cdVb5*KX|$O1-T(8 zl#-)^ebVd^x#Rnwe9!vh11m%w;pDP(cwT8K_AbIYULr614O2SMwywDFNVrJsC?lC}U244N z-$|n2aMP*_EJ=Z--hp&27BJg9If;zY>Ja*}CtVG}1R934zZ-~R6jpZI@^yRNliTX9wlb2! zpDvB>W)0KTJaohSCcB$6Z4fb@u~5LY4j65>)O#IKL#?ZoF(+E9|NAX(yn9*pDxD^Pofa0fww+d298LC$Vs~b4Lst+_$AJLdWIyj zfbk0PB2Q*hGJot^X=!ehXEh7)`-mOGc)<|-RUIX@eXgL$ttE_MBZV#7c=+g@%|(PX zGQ4TBb&D;5WV&pzOddG`kXbb(Yc}iZ5+J0b$U-|5KBkFcx)u&>=X6>8_(bA5qnr}i z=!OWo6!#Y;akz|u`jsNa>-{>o^r#rZ)Ies&=5JKV+G*UZ5f6UPC`X3~8l%`TD2(AI zYPbRVc`4o8>`7nKbuzvsB_Gei1R|k4hZZ!b%s69C-$5eD$J!Y=LvH^1c$_{jlB z_gHR5W`;qK;lvkBD>L>Rg4Ld~sia2#JaQ_MY3mm*MoX-YeG~9jk8>Tj*g@+_pTS2= zS72vs``&831$Rh)ehx`#8d=fOQ8e9nW?GI0rjH(c@$RDERRtGK??&796>3K_`f5EC zGp^W)T#0!x0v17>$hH%;=)1htZB$mnL$ay@!%^`i|8Z;fdCTV)Bk)XLUIn}h(Xg5G zKR1G*m?PhDJ?+Imj-*WDJ?Yjl%P=)d6I3|iEf{kn1Dl3eXrR~BAiM^5zDmCvQ$Vk4 z(O$_;*ZEQQ(vo>XHgo?LM~tN#tcwGVuS))b1nq3XU1b$^egu}L0r+G69e`N#15SBc|#-b&w zV&h&-2BtssFeKtKldt`H*UUe}{ccY&c6It?=wk%+Z{u!F%CQRf1T zfHKP>K2Dbz4PkvHApzb;8yG`hjvWzQkkL*Hg!8lWmHH>eVs*tH$Ln_<>Cs~X+tM@)LFeX& zY9+%@FR_e)$sarVnXC+o-UsM`(ua$mL70tbI1}Z251au%%@n?-i|WwuMObqOBqNRV z7{X9Evepk}SZZ=N<1P-MksyluU|;4baQ`p}!u9%S*EUQufewoX20#W2U{l-W%O&A0 z@8T-vVQh9YU0x6dBP-GMN?9%3$7!INlXf}!cxMQ+S3Cwm z#9SM8AQD;F+RB^1TJ`r|m+l^@OGfhEx}cADD(yC+uP3R#1u`85d(N7U1K4h~r2;xc z!%5z>Z?|mE-_+L48D$rc^^zDS6Z}xW*Z^8_UJKGIY$y&<^*;N+0=RU6`j#Ss5*d5e zN4Ul-mvx2khRnLslg{`~Dyx%7FiLg&N#}GF`GvF+R-%xQ^Jh+q-`$Gm5yQ*0lcnzs zy;4f`BM?E}k@n+_p3U~^-$pomU=IM4N{%X=kQI1VAUOgnLslu$zGT ze|8L=k=DW=a|-C@#lKSPi(Wz$6F0Lz%j&5dk`cf05#BV1AD)-8-{@{MPT&kn*_j}8iK{2o^Q}7CF3mUvt zICkH@_;%h&_c1j9*sCE4(jxFO;$sY4=AH&;b-5w)zVsiwu%OOL`pePSVn{V)<4Nj? z=JoM9mCOXz&`PMlPx`Of>vdy(O5G_mJG7;!J(5th#!nwEd;tF4N7rvKRoKZ~%_CF< zA~JdETV36^*+2!3+fx!3kH`GPe_*#KZ>e2 zSrX8#H@O^U>(FUoGU{;upc?Ekz!zc21V7*~qcwlcijPqOiy*oEgiJ|gS@*%>4(@UV z4sdgzOTVSI=YE;agu$*Xv9-CZR)6VvIniVek4#~V!M7O?xvhtiQoT_g(tBPxe8ctx z?lGF&Wa<1It|37-gNM4QgD?*x7QJgEc9cb2m}K3eX@BEQmOh=Dm0PG&{M|memqeVe zRdWS#mfoB}bqwdMLjR%MR9|$-Ym9%V`QT$xOCmz^8_rzWBw8ScKqQVOw2x&Rq=+fU zod_Ngl^v~R5T|3enkjW5Z8(Ez0qomV_{NAMe=}03-%>^~t9&l6F!eAn%=%_rkmPa6 zC*{o#kBmsP;QtLvdZv+%sh3{k9AkqFEG?nz(ew>9*zbJ7x|*em=sbIekSvW#@cspo zpeJ+_vR{;fgk1m8lU2j>L({&nM~;JEGi^oie&~g1PfrZbO8S^ug*l^M#En+}k07rh zW7NyRi}>Sg8yE^^{nK=myS_`$XqE5kX|j2_2n7DdFou-x=#Fut3E8#XsNNExx=bA{ zTsSmG%=(c>)-DQj|oMu+qeU!l3M z)0(XH#@XIbA;@U4%hzDo$F2v^qy;_O&k)oT3I&`q_(x65Puu{0#OB*6x)G(36(FVC zb6>&aS{;t$##Nb7{)j;Gu1uJEf&${BSXnp3$Z)U9Q7u}vuK)X1gH!4qL$q@Sl+G~T zNyV{B%S_UhDL5P4iabPll<;~3anLrRBnn9;#bGtC2<$%yw1x!xAGAKdRUZP43;9m--o~jW^YgW$ z^iOR7NNa=^RK0a(d`SOfevo7)5@+i8Q^HPUVbj<+&i4OW2nWM`^B1v++D4}GUtM?K}i`l`zj&E zYGl9!``L5E=n4WzXZ6Ge33JWL;w@xJ_hHAa;(1D3^vC#MXA+nPx}!s zl-!<%p7x5%w}+KQZwkT$7T2_-SBGDP(4ii# zx6z;`33<3p#TJ|jR9~sW^+*P|v#lhYK&zm2rXA7f1Upa+_3ycJpFhbOL%wjQrtLfy z{qjXGe0j{*DxADZ?2-0!Wrt7|cZH9AqNjyf{sTiOc)!@Dh4t^4hl-s%3|Ai`1rZd^ z#UW0z8G0QEG8$O~A!SoA7JRYN%F@MJKWG7^&`~hXvO9$E)>D3il?V^&03ki6XeS7^ z0BGDFDId1EFto+d3_-C%x4VF%=pC;!m4spU!W8QD>o#XYj4LF$qB(Q$#pcpv1AHhIb_q)}qa+6vWI?PV2i z6y~i$rm@Wp$lN7#vQQeck~sD?665i7ngtp%&^>g_wYvLlWpo`2GDivJ?O~DF@(hnW za$(iPOHrj@1or^NG2-317QAw5Mr`qO+UFcgV&S2|Gpqd&_b+5d-9JDn)4wd`pF1}3j)>(G=r zQr&ROjl)Xs$#-}5`diEjCfVe(U%yI;h#Yh|P!^5P@#X-yM!!k;1qzhuCB~CL<}oF{ zy`m1`kyS3X+vYm&3PLSFXdIZ5UsfagHLnDTF2&vz6*u-k_6gQ<*7^FK6TK{ind4fD#pI3jn1yz|-(eSfhdn}e2y}LZ3zV4zRXsz^8JxG; z*9jFKmcE`u=&_WI|GXJ`AWR9^(>c43eVztF7u+L-oOO9!PfAZ}IL7s%z*3A=D$b$p z7jnM-9OoS95Zg>!D(uxrZI|o~Bo*T44f3YVp?l^9YfKN9s|IjO?>W$v!b1l;Uu8YZ zX%#n(c(deE<>4m*FCLzcCjOFp>-%1k(92New>wz?-G_?^B#wrY=y!F?##l5Z)WW^1 zVYHh95@eWaBSBAu2!-xP3I%xo9H5GmAMaRy^ypgx_MbAgJo2B91-?4&I*aFuj#AE! zN#|AgFM{?6d%el;wzKQ6?KGAGtl5ZU#m|G~t_}=T<;wwBsyjpJqFe zk4uV}a*fA~6=Dt*A@P`ty~&;u%$++M^yb(>ww&{iCgtaLSSh5Sx!_;AY7KJ0Xen-y z+JoooINS(a&wE+0k;f z)w*{OhJpxbnjp+Nxfb!RgtP(h7~6HssY)<<@Js7-0R1ElabMrXO043q(p|~jtKLi7 zODn(^4^P5fIBK^vhBTr-cEFv}MNcobRi9>U2&Q{yyMu3;`*zRwTMM14@k4eeMgm$mw!g(T|phY(i6S#)ogYEn3O$RT(Fd@}Kwsf-EdA3f8W2#Gd{Rq-w zWS5sR#e1E>POdR_g`maVPw{B@y%*Vj{h z(*2x7CNz52F(Eo}sp4F@=i(@ADq4s$^yiynOO9g+73#(UQLSi}^TKmNfpMh;E?E&v zGRV>n$M%|UtAfX@fkBiWc+WF^wc5$^mK3e+?E#lF@q>>)_D|AKp`Ng2}0L zOmx7D{GS*2GYzx@(m>9pfv@_1Ubhp30;-Cx@!O1~IBKJWtWkrK&`RRX=k)!3Z8a&J z*y8OW_ZZ2l|KmCS`qeop05QHrcGdTfnn3`97g)q-0O)>m%I661*}h$u!PZnvXQ$Y5 z*vo&zlOXaQ&OVElvO9 z3jdm+(tM%_s{IP~1?N4n3jVyKAe+<-L3;SOm6~RHafU)e>szLwX2lzY%7~uBx2G0L| zL_UZBBZ4y)X#3ZDBmsvhc&@~|SCrHp2QVI5hBkH!<1pUaepdSdpzC(5_?KD`s3-K{ zE;$~yEN%bs2qI^|Tv%JzT!`9uhOaz3pX_Eg-^@!U!=1YFrZR7l_ds;CaFHSP}u z7#UoZUyr^xe`Yz-fy3U#guP3DbZEf)$MYJjQ0Bp!ti0B5RR3e}q`(m1trH|9I(h6w zp0dB`7kEz;zCPXP0c6qfo`?zyPF29y z{Ge#id`EDb0?5`fF9G;3!`VtL!}HzAGXE#qg4NoO{+R;DkbkU@3F)VqMxHMJYZ}@> z*RxISFCK5k{X^@=g}$bZzPIB7psDNt^vnU!WD5mY7EJd5P#I$<*WZs9AU>$1SL6aH zZo>$g)Y3|0fJU;`)bU{3ydKHKIQLZv4m+I(8O@yh4EPrz5&&Lj_+r@_O2^N$7VOARd8IQCC({#;=bED+(}H{*lsjsX6VZamtq z$zJUd7d<|kkH z{f_W&*_&wOF82vhq!55IVjIiJToYIJZC&gi>ncwX*f7pfioXB#kY8v{0E$H@a4~M3 z|I^-;$3wlY@j_*+Sz@eXEFo(`%N28w>`u9*$kr&FB2h>vyN-$wPFYVex3Yy~E7Ks9 z8cRZ!jE1shY=zK$UerDFx&Pe1@8|f7`S|?i{XOsVexLRGK0xQmI+VNUIr?hQ@0~lz z<;Xm=b&970BS#%dql1TN@n)(R0eMUFYK+gJ$KPRxbc^(|`x1RA5;>PTcme40PZxrtFgdLQxiXtw+Ly*Am4;NtlB8W-jLa2+gO@c|B+JRpy-&;KI0JcQ72A5EG zXMJMhydCp`gkeET((ZfL7Nm-~iqq2-(er#V`}T#3$yG~#<>fwbSsAc{kzK{Eys|2n z#vgUWPu-ZErs`cH;2Tem?Gv^yDSA4fFL<54ac(NPT0F<%_Tgh%^?p+kK(OGZU6D^u zA*OMHtr_BX>!@$KrKL10F@wh%yoYq?8>;f7HwT9^=axW`b9AGhM(BP5*3%h{z+d|r zHM~~NU)aw?+$d2-1f#h|t6+X1FldS5ORTs#Lb>sR2hRFR-6}Bsus8%9An7ox;e$1~ zpEFyxZ-C!SnU81hOtZRs>)qYjmp-QXQCy*kk^zZPS>9{M`pK-kT(0RCPA-^)I`eb8 zK4Y96xkuWVcqK19P)b%W11f(`k^YB$-+CB`9A88?mMdnRwcni#_Wu>WIX^ZlyNkg{ zu3kKmNkhd}bYEZFL^oV&8)v|NXH)3RF>XV3J6D z2BOt3Cjc(T3GvQ?(e%k9P9YmM=27(&!r~LuwSeYzcn)|0!oaVIt*RLOXtA?bP`meB zo4U_XQ{nv3?Z?Warl3zEYJy$6;|svK}T!VKh*+hoWmE}^}eNS+osBF)E9mMQ>3S08?_3(nPx02xuh*Ybz5x0 zCH#6G4!#Yqrr3SB{~({Ejbz2^xX_w>a2_!podGQ@jIO~#DoS~-D%8^CU4%?cO1qt- zBq;EbTUM}teCl^}!(HQ?cgsHE2 zr!a2HFeyV4(yVRmEuPRAK?A+B>?49kIK1>2C)y62&1SL@>TF;mBG*wZR%*ip$QjIa zQGVnqP}*!;O_`ZaxT=wz7Jkqb%ZyG@r1*6pa7r29j! zb6eAduY*K*AwKbyiF}%-0{t-QTX8CzETd20Q)z5g1lH&h^sGlA7>TR?5NE)H5%VZH zFgGAv-PyiA_2TDp8l-A}>x8c18erPJENs^2eALsYnY9`N!6tw9vn+`N&SHV!73Zmr92q`f29ME7TQ=njEz%?Uk;!KVJpW5v{* zi`8vmHP+lU^`fV8`B0y3#0KGGg?B8*iF+t6cxIEa3HCSO^3c{b1kB=3*-kWivDD(u zo?E3Cp90N#uR{#$5Kn<)M?%LvZVbZ?Ai9HAhn3C+S5l%B|nlH{lyD>5V7OFBNP2W9W|GMn(HUg%E& zr9sBuuYd4>5XE6HoOAaiZM@saz>Gjn%%aS0HA1;0%K5#LX*R=61A~P0l!f(qqM~*D zjoFemOEcPntpp8_R?HU+4Wb=3IBZmn@K{ygI?xiT4u?;^`yu z>M#f#k8M7hJolyVklxt>l7B(n?E>eVb0-)h@t4yq-%`veR?I5!9(fJ7|24Z7cnI2aJI_`zOxz`^qlw1TCb}x% zdVzikgIQe-Mm1h4C)jWY#>=MN9R|i=KUw2xoMZIUm{e)h&RYiD+>Lo^%Z0Iut!tT) zA9kbJ3i+}oZ53<2to>#zbYw}*yqO68TKOUmj?*=IZ^+pJVD8q4Z)B&mued6}eB%J( zlMTnwVg`kiRB@7;F0s`z6)Oa$!wUw?yyENoT}L{+^-pfBm2KUq{20QEEe)7V_}Xcn z5Z@rKtOxAbUu$s7>T)Wif==+eH%UktbwERW6x@8R7H313!Ob_21NhS}4JtW`%-Mh4 zK#jAu+@0-to@3kpI#k2OxO08aHZ(|L zz1ofK#OG{WGS(zZZsage@yx&Wd%7da&)CyoS$6#bR2TSw08>`g02<;XDb;*GHftVS zRSwwVK#jQLzh+bYfx*2=f8+*dxNfL!WN@@@Dd&aO!} zN9dZ;&j1g-P6!^QL<}!`s;<86r<-IbJyC!?FP0D7dqKbIj(~R_QTo$B7B<6B4>$qu zCWA}Nv&_m4U$z@$eJo`XCE*{S5&wZ+ zCoJYPIF60q!U82N5@oLMl17v_8XUMY$EQyxfRhnS<*cJ}OE!GS7ADJ4)Pg5t3cF6;*1G3jx(}khICxZ4rZRKPyV6W! zM0cOfT#<^CgS1!E_WE#P2Y>wFklVpZ#C9jKH6ns}!mSf_ATL!+XZ<-$(*V@zzrQ@ftwGEjzC*i4X}JSZ#7S8j;@jFT7VuKK~H$)MABWpYejqj8dhv$ z!~n{W5%crdk-cO~HRN55jmXMm zuUQCt^Gj>e;8{y@PPjJ5^ta41)69T)e<(x7PT}W{{N;;&{z^y&2>UNdF)a%H@83^j pimyQ<{+8R1xM@{c{y#Kid6O7X0OQ7ARkjX(O!iqC=Ic9!{u|~?nSKBO literal 0 HcmV?d00001 diff --git a/example/gluon/lipnet/asset/s2_bbbf7p_000.png b/example/gluon/lipnet/asset/s2_bbbf7p_000.png new file mode 100644 index 0000000000000000000000000000000000000000..6495d2fa5b838b46bb4748e391453bbc34e21e04 GIT binary patch literal 35141 zcmZ^}18^nL*Do4OY#S5Xwr$(Cb0*2ewrx*rW0Dga6HRhr=fuWj-u%DszI(r_`(9V= z?%ivx-(uCS-m7A%-rv4<8SM zgV7WmuYpaRL18H_)r=C0sJDQ?QC)ptHWWG4`;&66uXEe~;*VoNzUw8pb$ggq$Q3nmnwC8u|XQczk4R(VY?kij5qKDcQ1F8tOSYSuEJj<+FDijeI-WB~N+$ z?fLKD+CzRL3?c}+FMCpn(FGwl@>WI`Na6Yr#djFJDk_;~xoG5dbW)g7FNa*T^LIz~ zD)~nx1Vzzos^R01KhyM?%sC*+S)&UIXH{hOlUL9?c0B0?&SsTD$o)A2VZLMh}08Sg=)<9~HzdH^nHBv#3q;eqB8b!^{V z?5Cm2XH<=_g~o;E;{PI7MgO5V$!5Nb&$C3GW)$n;`L4c8OnV!Uy~L+K6U~pyf|fRR z659tn$)^4o$l_yOGVWo`qEE=l!o(0B-{`BB_GD5(LVLM7pl?SSHDAh43l~`;=)6ptsB?+TZtV7Ig1Oo!I* z13XfGB0re^CiBCF{KQ)bA-jPxpKyXsW@znVY^h9pHe#8;xl7e-IlY>=+*`^C=;)>g zO${$W2A}ovy;tun*<9gE^}c~J1>qqmu|r<2JEHv|yvW~E83s3zA0dmWAQmjAagkbW zolnomkUfCbArxwmgcH41@H4?UP0%VoQ2@|E=*SdeOnQjx5D&z%o9(E-TT)=;TeM2rYF?8eNlmq zox(VLI1)P2^Qb*yI8vEJnNpt6JYqfyJK{W4JR~}7oDkLgNq@=+V06Pq#71Q>Qo}DP ztrpNyWytFZF;MK6BhemVAf)5K(Zq?vNn&_nNMxj^+n|%CuUD(l5-6i8DY1LwqG2xR z$neSM5qlR>{a*Mkbk#R|pL-0-eHQ;V(2~cY+b!3v-);KZifBA)NUdz$tB^M(sohr7 z>`?8HcrLOMSHpV3IOm%?Uueqp_vPc=p((f7Ap5sMdzi!83e7L>U*A_?&i#L}RuBFv zw(qhpcL4w5t7Wg0U-4Qzw7>c#RP~zQAmv+CwLE9=4iQJ>V$Ndt#dmkN%u!tIFMe=$(v^L2%l{<_(OfSKg;XcYfmQOuT2v6Ei zVaU=l@MYod8;}RPdrf0|Z9rOC!EvVXgTx?#9R3P^Za!f{yQbvcCZE0N)(+0;?%(?R z629_o{JRwHD<0O}t6d^K%kBqVBp&|fE{BEZfg^u1q9URe%0>^us>)6n=2ckJvl?|% zbQ?QEean3#o+-(ki>!)dirh^NcQOVF0`UhS1}_7{l)P#3^yi76JD*LDc1Xlf;nAw$ z0mOF1bjV=%(6Em%pD!rPONJw+zi^R5bA|*%$s+N?ts*}#V^L5L>XE-=+7M)ttzxO5 znBs1ue~JuKS~!k`$pPiut#s3!v;wrg|BA9}JYBMrp9#&@W?!?=w*1z7>E6{<-6iRc z71xhSvP!=2&gPLdXt34UY_Gtuxa>aimDe(aYeQ zd$PK=8Qkr1_wg9(n%mO2TBbTFUn;kqoSPI(9oI0@oYFA46Z(eJxAr6O$GfIzIZPFa zN}fWJitF4@TPmykea4&R7tl0!W~1IVEgJ?lB2`XePA|uJ1mv(Jgr@5b-Qsye8g&p-~H-a9;Cp3jbu%ZAMh zg(Jm#1zDdjmv$ie9sRa@P=j@EL;6Pg<&N?e^ENX*8k6fzS+DJ8AuA3&&Yuj@j9G05 z?YvqM@DaGbecsu0Iq>n%=({M{E=iZ+r6I_m%v93{-Jk7!H|=-z@4_N{(G@ZcVXc7E z3-eENd1F0e-~R94zlRwBS7o}?xA=UhJ|||&Ok_9nW^!xt>}N;Ef3n{;pNP_a2)POE z0@e`5Vk*=8Xlo6snWgL199=E|h6;J*!X5qB>3@GTpjIVjn$86A2N+y<9T8M;8qQC)+G*}y>v5XjXEtC8P`LpltA1R7tu{?A==l6mGULSDA8LJvN z8aj+ydsqEHPkz&l9fKVYzaKlcn+6vHm-lRBcZLUAJB?fm1~0wemq?Ba9RI|l1>RT`IkRcAM=*&jaso|&Yh+QPfo#L1 z-)?(PpIOUbROwWw3LQl1gKQqb583nUvnO#yNdxG8*#Y`5Q6FU5jmV}2AIq2X7YFPI zWsMPyRom%(+TZ+N3TAs7Vv=KIuQ|6>1L@wX@0zc$4m9UaPUlr=JqLCN(u26)e%~_h zZZ2qz{Ynp}wwi5&qOR458);D(b z%iiYy2lgM%f5ZMOuKzYC^dDscYW4tYCw(bO#mzn>I^M4%uA4t9b zgXCl5`Jc%D!TBG^|9B#x>S1sFFG>GNg$TP4>;FajU-Ck%|H;$;$lHGx<-e%^sv?3Y z#QMK0EQ08};jak+AqF8YC9Va4oawhuqmuWiJV|-h)|b`wRh>sMqtj!;3x|)`q5F{g zoKHMK@C zRr8fY+XkMdHR&0p@u0~*FMU-txN~<6dAsZ^X9CJY1Kk=sg|_0>@OP+8Ty!2Lh(he) zy68zCm~FjAZ1*L)Gd5eV;0WERQd6Nc?e9vh1Z6Gc$7|a#XGWK*Ttb4c|D@P~G*vBE zT?fQg%TtPk_J7pgzRosv_2M{niHMSPxIFnNZggl%V5?9ct`UI2S*(VPGJe(Xny`Y` zprYf)8_7P>H9XhY+U&|(L|G+Z>e%ht{i8o0j_&OXbEaCPTDnblN}K;5_yw6F6eKG* z|3OZs4&F!t#ID0mv7rxU-ed_ve2%RchP*bdnj;(f+%^cBv_3Y8Pn)6aCT!E}t6+MV z6;o8-ZR**a!oEVb6k#c;)8xfLmJkM=VP?aQUN?DzDtiJ;NFXwQ;+)q7N00r5;=p-+ zGH>Z)s;wu_$q!qr+JH_T3dV!y2|$c@tMiqMPb4#QL5uQ6x@)&01Cjz`59m7n*&ESJ>M^ zl_zQ5kuW^u^oHH35#%SLU0}m%Lv_!Ne-wO%7qm!7q$7h)8`Ar*+U=(IRE6Z7pv<%P z(~E>=)f#FWFo9DOkpN8cOp+=?d%L}>UYAG%pMj7fE$stgN7kU_pmvu_K z;VqZ_RDl-SeObm{e!sChh8eGF;=*{^bp}f^zu4Psk|%wekLDl zjqL^$hZM1Po@r{ZA&*wB+g+W3B!%!1{^8gcObuj!;ztymtSc755}gVQqYVRl1P1s9#Os^8wk&##*Z$}@-f6%P zK6fXG0`zG$5bHIuPBlH#POslL%M<<7q8WFgD3*IuH_nJHU#RzBaLR?o!C7JxbqiI| zYtZ??3FJuQJ<@luKng4Jn5}(e3`3fas{Rh?cB>K`&3r>IkrO zVC-2;foK5KsI9DgnCN7b4&;9^iu67J-1+98t{LI9-3iL`2uV+qfd32&RO0<b7pzE zZa2=eK!R_KH`yEGPsGUst*PKdw!3=>lTii-6g2#U3pX4_C*bF=jv5_}k+=-W@b0oM z`-$RAJ+VmssVpyA2OFkz92jSQF=U_=McKYh!H>|UzAo%_%AGx==WKtb@6xSEKk(@{ zHGF{Bo*B0CQjjy6^hje6**bt!9jtVP)7Yiu8sxn^3)tr2lzw;UTZ69bG(mm^ zeFQFfovxAxx%*K$XUVct{gLcUwZ-qjzu9lAUK4WALTXchf6H>1*{OM9IP4Yt30b?y zg(1}I%iPLk*)Q*)fe|D;?A9{*ZY5kXuUA%!B5BFVv_|_> zY(c6x683(ogg)e~PV`EYVxgc&aB;303}RtFv2@~K}==j54C3>`gB5<2ujd~GJjWRXD-?9M;0l^safuo@;8ERGL> zS32^`dWqz`(XLKTizHa5i0k@49DGYJno*ZM$;Omfh7}2o z^j{|!^f7;YyE9xPI-@hYlO3ITX{7(nhuT)_9j_~jp|BVm(jbN5V;9vs*2(sH&r8GT z{a4ppfivwID;&YBtobffT=<~^uTT4FP?+mZue>I{X%lmxHae1+bp{f-ius#!i6FP# zqwM$<9t6Q`(i`{mNsUs^fsCcbTg`ba@I%H0>K>rBK7Q(rc=0Wh@~E(fVXfRj3&$~D z%OEasvb585Y0M|Gu&L#DNLjNCv!n3fIENxz%}~V)yiuLu-fu}O4-0Xs7{ajSeR)NI z!`?3~45Y8=xuM#*@Qk*B^bPhmgkM7Y1;%P6CQt4{6vLo(=RN#8ys5X&9u61(&dp$) z`@>N^{~TpdFG=|W_ZpOUd})+?s6k3P z0E%m`ZLRgDk}X`>38EkqA+jnVU*3NSNdzl3F^Y`ZE)ch5Mi1UF?Fww$_s9uCusx;%msihT z?AlBRMs&K=t%nh+^7+x|^9I%4kmWrGDpgrq2L&!48mZbEa6A>m+M_rles5p4iwF1J zL)8=(C%jo-r+Dt1tt3jTH43_!P$|Hda!FfYj@C?~v)UU&QZNGTMz|(Wmqi z$a|Mu2O$g~GbEI0W?1KoL+;cwYdCv-vy|ymEU%R!C1T_lWn+7;bdmk&Yyj$NAOx(xfH+0TeBWJGlva> zeAzP|3o@&xhsSd*$rbG&XL#fi@}JVs^ZyM!E$C*1n~D{^O3$`P|2mqFnmZ($NR`#T^IJ; zjZ+t+HD!_syn31h`+k)8A2k+5`X36AE!ot)07eq00Ne;D6Qs`apfFnm&|_ZX_}^mI z7Vl`$3a#wsz=CKfb=f+vB9iC6m))w`b8Jb~^ZsxguOG%o6;3TnT9k<%b+AT~w4$(* zMMXUeM=sPajW0FT?4l1!Nu|EKP+G6CCYlsu=sn_)A9DiZ9B|gICI@0-xCYN=dh^V*ZhY$y; zc7O~W-DV!KAa~B})(pA-9~*aK99ms*r`w=jUhSUdS-Jp&DeN)Ks@RJBKX_kt`gXMC zTefTS_!E-uMp2|3C1F;{k{OE7Y4J!lV=I-k`glAyN@t`He~O^WKdd#tDHOYMlE7(P2? z&^4ctr_Ed$7ifS7lQ>vO$Pw3y7*u*utv;Y4Nv$9=U!SVWzELzLKyg;gvr;|6zOBK@ zW#1LPmd=I(v9*rQ*kegoZNxP3-op5iUXbXTGG$<5cuPsbNT>2!Um!AVcNCt!0KL27 zv?7|O3gr=+F(4w%mM`QkLO(9aOZkL%$u8EJP-!0QaeZAHyg8k>mknB01-uM|XfR)^ zHhbv;M@L89PISD@8(A^v40)17NiyMBqnLeSi#?LNEb{hO*~STHa_ac>C77<<&9D%N zgb+@00&7n&secX6ZD5Ks`dF z3en%n-dXCAps5iXxeUcxYkOhbnwa4FOI1ClnrUoffK#VRgUo=NK%|%^(f13`P?Kcs zVO_-Kgg@LZk#{n}j@O-= zM5N4zL>U7YvQqTkobWX(*@2_wz0mxmB`0(F98veV!@LV#tI~xmot41|@p?eU(yh)> zGE^{Ig!-oeZ>K~J`iEEI;qN(qT;xV3^XV0&MN)uV^-&&^T%+x|xrJxta~~<`S!}eU z2?X@CaTBHy(lY$8{Gj2$DPD{`u+a+|v%Pd1@2L<(`A()@R_+=$TuWFB^F78o+<< zw=N!X-7p^&$9o;wPpcZ4cd5L+!B5PCYKq)i5N1xc&J5`7LyCN< z75-lkJE;}8vsG}*q!etZw{iKtN2u88-K2Um>2|GmcaH2Yr$5)XVcg>D#zlN}g*N}x z5mnnO?8wWOMPQ=&lJabLp%7S3a7@*ChoNvbM;A#M`wAB-<#34eO-}YCwIW8h; zS*)<%5Y;rr9WJWo@9X1L*NOM=3qoLeSfW}Ww_ilq(oO2NeJ(R-b=UiETWD_8-Z>wo z9x#r!e!*@%4M49!Z-zl{Q6ihbyk^RS&bInn%$(=U;@L)p=9nPJnUfRLD@gX#I7rf= zI%qE89#QIo?C(R`n+F^Y$vfcgSpfY3?VaA3bq!byx5yIefbTv6x(0j17CWOhTWj}5 zX;q^@Gx4sGy{&P3cVL`VH#F+LLv4exD7u^8wAQ_|xcSIGzyovNYdgT}p<7qC0GW!) z07)e1K+8HXih8HybelA>N``-ZNonGT~ z+mB~W?*5s#xsoHjn@3HGon5`XoJOTyx_TVFDzxbD@2Zi1$|UOuL5OU$*QO`N))T}DkD44#P!fkVL)yUpy&vX}G zzV+|yTg~ZvsE2-4YcOs*T9!s<$Z3NH$hz5b{hyj^BJyle3|eT#Xhrr#mE=n=0}a15 zC?eRMPsZ)yZ)|uXIPH3{N^Vr%94u0m#N86!2>qpE_u!-@ncTYF;8K+}JP#9cGBAuw zAjmayFtm^UW!5=un^Sv*c(*#&r%CQs8Fg^!(4_(a6$WHIEe1C`yf+f z+4t5x1mhc69lI=RJX&Pmp97yyn|sHR4S5J-$G8U^SGjT1oMUcrdk-I>ECK#}W>77o z)4!k64GjhLnv3@f^|jmpRREeY1fd$pZ2<4k+i*d%Igr3PHs z>ym4Sd^D>2JVW8X;P3Qyzve%SEon(W{|FPw@O;b1c`w9Qqo6m+G-SJ#caQO{Kzfz8 zfw@S@Y%66M9f1}gS+@Flhh=!gEuzzVBF}gR^~$-8`B3(#qNHaBOFj)9562rrb`$!v zY&;gl)JmK$sNY4fT{v)X1fJb{Z9M_PD6(wMIGw~pva)2;m+|!y<2LoTCs~-3@bSh! zZ=<fV@_P=LIxi9}h#a8M)7w+Qm%b zox`#G>7Vr7?EM{v?_mcB-ko*pfAw1T`y6e01-F`8KFgmKE-h0qU^W=L#5}&yx_US7 zF*?G4sM?|45|h1iha0N{;Fc)$D_)FNxHH%i91qr|=!e{Jc$k|`eteOTDmk1wPvehp z4)0l8r;9$7f;JhaXPcJq34~slMp@!ubg@`ddbQspuirll1x5zWTS8 zy}I!W?dYT4Gul+rR(bVi#ow|-MkvzSGG*9tH+$ioTt{MMTJY#!Y*rb9wzsGB*jk;r zrsOBj9(&E3=MTS(`rW^w966UGZCltlj`I&nr?Bde4QaR(XA5Lpt-53nGOr??u#kb6 z_yVnb*Q@Fjd#(k?;#)5}k%_iZI$4{R=0(X+)nsg0o{`=@k~Q?uoeb0yoFqIEaO3NB zLz#D1b4f6RLV1-h6U+^u=^Gm|H>Qh>DZ6|ZLErOZ27vz(Ts;Wzj#i0wItxqy7MGeqG)pHF{%#b#r@Hy-RG04?ZNt9)_}t&Td+<)mmx7u-|V)?*#r3!r~c%c-3NRywa4+ z;+Y<|RCrU8jwKkn>C@o{8pQL~ZRauTUcEA)Ar0D^Yj(&0Axkzkm;}6#Awuw!&Ggc@ zIw*`=to|DRx6ipq)PF1uW6VWl=x0x}sCk!6N`R5?q|=&h+3T%Jy{Z5)kxMDliER}f zH=mv;jxSz?h-c-@A?1Mz(;;eFTA8E?WL50|Mc}BO_c4-~Al<{Z$VJLbPmJg{*rKQ? zAdF;;KdRN#koZ)gyh5ZJ^$*#G4r&hdroX1$q7falUqs6=37aAF20GaS^4Pt%eTM5R z8l|BMuvwOgUy&PWaC0zat@UEsA+?Cr-QZ3pY}w1s~bwteV7l7*VG#5S&R+&uLk=8QFk*sl9?RvNqcAC5{C{ZX1r>*z`-oBqT7XDlNZzFp~6_tbooZ+2`RrR z1ZCbMdQB2-Rp6Eoqm!cJ1o=E%?i5(|u19eE_<)fx!cvUUJ*2%s(t-)J#50htxns~1 z!_GgV>Oj<`NvLe;a8Hh%R`NVZ%sJafjt~^TY|tz7bcED_A!PUs;~0e5so2&lY|el5 zk>8v*?`3b5Wk$TbF?;1CC^$g<7nxdJ{UxOJG8;AS+xW}U)iZ)|)hE62C(MC#mc0!p zc9txFlc|O`ZgUf_+^|(}EZQ{aQMu=_><{plcBKTC^vo`rNTX>GH4}&; zAe^)%hV6l@r;=U4p!-L3&o0%&p7o4kaU~mLHM?)dgRrE<6Yg5$RaaGVo{w@L`|o=u zGfbCD*K^qdHhQ;EJTuI)bQ$pWEZ#y~V0gK%4)0VZ6eFurBp{6S5J%+exExEg!+BB@ z+osT;^*##=j&TZe5?}rNJMfTWrU}a_gR#tpn0P>}`|RP*$U$?a6Pr*&rjvMDMgtKc}1+E)iuq@(#XvAEPTVny(GwK=PD3u2yVq36L3?>My=miEFs99|5WZWU(# zHu1BFUPqidd-lG>hcdE)8n<4PT!r^CJCEyK*2|EYr^!Q~s29PD-!fHk;|KGNO^l3M zw}!J*8XiHtY4IB!Ip|xge1g)BKo&wAWt#$QVzz4ls#ApB z@>H2Vn5qbWIJDPY!Wid?%V*jT_I+>0Y^yW|+(zf4TTszWbPq&0$d(8)&=A$@(CJ_+ zIfd?9MlkshaLOWf?t&3)TI|$(MSb~texuM*)x8^YW(w%#FC^SyhsxT~;tA1M-ZhW! z-%XxzD~thB2C2szuUyC$jLU+(8mEox=_%V`R29|Q4i0F~JR0$Nd)hqwTdj2ZleCYr zWqWR`aoiHAT?uR*ATJ%-~A`U02wvTZqQC{0(A%aNlTAXQ1OfkCyS$7 zQip;vTeg4D;ZHp{3;N~`Uv|&1BJs!7ES+LJj=i9LL?rCt2m%MU`f4YU1qCY=Li8gf+_Y0Q+du)#n?2Uiw@E&PzkV_f#_9 z^}*?HTcurz(%P77^u9U$y|6K$FNrWdH&-gEQQuu)iBhzr%9*B?q!_+KvJYq!Ofax7 zA+kB!x+xcqOxGVg{u12?5@P|m19&OxMZbcUA6_3%zm%RYV9uZ!fw(C3#^aK;DzWvN z{=lL*P^yuxeYpweWpe2&WnT(&zh24rh@}vien`ntW#D=&5*8%(X)+mnYy8W#)$;m4 zz`0C7gtR^2N95Q$`EpKrG>qOXGEolc_@kv8`Z)M%vAI}B--K6huDjxT_JI3E-|QAF z(oS>cLg3hAUZ9QZ;*MRr)~@}ltM`Ym`jvCtB{NrJbH2|XJI6u{(=s!I-#NIL!oNa) zN4%fRY!X?FwSql+PJETDOo%`ADEVgk*@J&Xwz0n_?HY6eGeqy`)_>*HrzxGMh^3O6 z%)DxnBwY$DC)*8K)R)S0#z`%E3yYdlXi9Um?!J|m?)iLv<7 zG>VJBSoWun9MOCPqmwjtJHr;aINAnIQg*XO@iqn~SQMDMcSIV(zIx3(#gy}ZHKPa_ z?nYZ+FFX>QyJx?m(V1m&2=T51mPNN-CzXon-mb$&R!oU2c(u2N8K*n2=eLqpJ8|a3 zgoZ&(&qacSu@s}ZxB0fFD_85tI50e>@m*t=S2*cC_~9J82gkpBk#0-;sH!uzIb&Ua zdrb6b*pTRWZ##4*pe8xQnFUA*=gu6}PY;%Nr}{`7CP`8e=@)M&LN+mz?czJTEv?m; zJbcmobAqRcFURtA9f_kk>J|kT$i}iVLZl+CyeDe||3fa3ov?*1T=A^dU8~l`{0qxl zW$ea*29rTK)%ezayWL4WSI72zlTI}ZybwY6AZg(Z>cnN<4zlUz@n4Iz`wh35(i_d5 z# zH=$qcvKfDag;(hU9M3p?3OAw?lMUh*e!rb`n@2f}jJ80@VX)0nlCWF{7-hMPzM;4( zO<=G>V#!krQLlJ0cJ={!f~(XSNXwX3qt_8|`nM1OnR*uSU5vRco*<55v$N;2)B=6K)bPk5_c4|mnAv=x1}d;CD^}A^-S5^S59T< z({X#?=^bR?br3hPMGPwbGwA-~Wwvt%ecNmeiAZC^=_cN`L!|o!d^Ekcfz@GEGNwp_ zo!GnOckQdBH4Ij?GFk&6J04N}eC&CpY1JeV`ZhdH8r%M_dED#N0&6FL=)emO1?Woj zT4Fkk%@mGMJ~8%4JIyIK#aR6L)A$*2X*IXdd@VvB5=r#cEc+;@VDIDbl8rcM&WV7! zO*>?{-o+r*39O(}*sYw*yzOq_HQUN8KOwy*x#6)S<0!S(#8A3dZE~41L$S!=AZi}h zCP2lL3A%rPOm@QA$^Pj`A}+3Ar7&S6eq@~0r!7&Xqrb$9mepdg&J~41qPtVaj=&h4 zp>Oeh&h+1&_PXo~aJ0$)GZ#d6<@e;i0h0%310=Pi=Z!XL4w{GjOAEI8T*MP4Efn@A zR6O^a{K%YykNb5>+UKZWQ>$zK2CC)wrfJ0vZ#^{&*YMInl5&UqsQ%X z?%eq8o2cI5GH#Yre)f$tWlvCMQ1v+Mbt#y*nLKg-&)c+XH>6Lt))11Lq1XH`>PFeAli924rB{vhw zH0c^07%Tm8KInFNZ(Q^>-y~@kHojh{&poXQC(A1$PVviGSHrzxbK7^TOJ2SuNz5!p zoK4LeVYDrW7i${93I16B`K2VXvt~h``O;SRUh|?E@D&HFw7oIUhU$6+?pR2lx9@$L z>IV0{gIO&l*|WRNoW>F=s55p#dEaWHZSRATRQj`x^k7IU9YaMJvR334!gVS>$T-V1 zPWt$`9jr~9pU{v~-;db&P{C`ym~w+(dYoX^CocnswNaR~lX-qzgpHVP1nu_{LQR+z zK85$YuW(z95XD2eTCt2$1~YC*M@)<`2O_sAlh1~x-)?YK%sjemE^jrHbCV2kA6?~e=`){?$$^3>&JHD- z&YZrgG+!c>icrvStIV-<5B*|g|8fJPLP3K{l za=1_aHj8Z?`%qWuCpH03)aXpAY_gZ+;#WY;$tSW*rhiq!MK3w|93ol$%VQT2@CsjR z|C{EZpPB5eWX3e$^Yv!s;yGv5@abzKCf(+5zM(YS!k2v0KHD@F!9ivi5e2Wz>a^P* ziqe$`Nl}Be>jx9%I>7H-y|#KvwyVBOg}Duf~D$y!;yV9?TfI=2HzxGRGL}%t_}i+jsGg$LR^QN?c9o z^Mx(-vA3(#3NNE*B4i@pUxilZH>z5Vrf*)s!q80-Vc*B>&vev?ael%+vL1+A)cTd= z8|wrR*_l8k}WH}3uYc=_}0<9AYY}vEl0q$W$yOJoeVxpn#C)bcN)zZ+e)EU zD|KDX*HM*j4n~%-srsAXgk;8fuko6O(rDY|j0^k`Kkf-MILb4Gl1Lm@1q7#n6q+`L zVXAMARf}{Tp&_Ob%(W+b(5BaJ*zAmpujsb*=-66-jm=JvzPdImqQ5N0Ycy!jMIs07 zN~Hal&B@5*878RnMIzGl1W6%>RbHbRfq&_b)e8|6QnjSff zew;JrS!Hk)kpgapxLsI^oFDbnx*li?7nA6xJYHWh?d_ZrBDg$X@71gzNc~p#K)CplK(EmWvIk(`(jS8aU-THWVt+y!8xnkp-OVA z@!COz0dTeSv@Ak_Jo#1xoUKfRG-mVh*SXRnTyVu-qqWlJiYb$7Yp$C2jZnAef2s7JMu?KJ8T#N?l)GlxQt6oCm`5HB z9FJ2&Dnt{oa|=z~9(E}h)UlXK|D2Ncs6G9QCLz=@T`irW7Gs0&ym@n8!Oevp!+lgO z8DU`E?t!J7Oywnw*c@uzh9#Yw$}79S((WxGPoBDk*`(vb6sz$CL07B^FCn@ za5CET?FB0x9IH=tePbbxn)7;06f4x?e|v|ODVd$aQJCsopE`AUz*_|4Kb-mgo~}&D zRi`!N(@j{Y!3(H$m=t42B_qlyK-X?FFmc7*d(bjSXI3Dr9hcF?ffIVh(@7{-Z*}06 z1t$lM;I^bG){z9b^b$qq*=FfNZVv_)w2$)V5eS)BA}S^!%(;H@6Y1sce=fVf97)LTC6UwS!zq%l#ik*PGrO$U8El4x66I_AOs9% z*O}FZ==`vURzNBQtK9e-uv{ z9w(C`X^{($s|Ib3UzAs&K?f)FaGP9Lkkdpg(+i0?uWv@{)@fifnO7IUT`9NLH|&5q zD)YFxd9T=3cg^c2!)}7*6niD|aS;ojI0T1_9sABpzX%kF5F(pJVOS^s}O!~pYB7x6vDt%9dF6P6c;^OY5$mkYBTYv>oloYAeM zG&n%!EsBhlbIiWN(rT0ojpcBsVx%v8#1Kc6;UZVf!2z3rFjYVFBxV zzb;)5gXBG~{NN9RLM318^qp{MN2O58MMP=?`yXy{LvbQO_(Oa75xbXFMX2N8d&3VT zS?@mcf@Fl4#~bHv=0Xi(5&o&UQ##nGONy38&mK={X|MPbjTmFkRmu`qU9wMX7b~r! zsUHD}9t!DzH#hV%K$@=I*(ON40I|gWD!;>No+EvrG01?ZCY1y4hP+OZL$%3cZEc%L z$pA1We1fx}Xp-G?GG^EntV|>WNr|#a7n)MW;EFu4Hj*Kirsgb`%w=7pXdq!i*yI5a zvw`9G%16@xRVv>)_c^CR8>hg*-{QP37=u3K4LiZEOksk8kK5EejMcg1*T*;|*+sf` z%_pt*hL67)bI|n8yg@bs4N11LXE=^5gyGf66Ugwx&Tz?0he?nI%TM^L%LyxdTY$ zTZ$W#qG77`OAsMq?~3Q3LIzN;Y&26l-OapBE~ahiWwmXJA6d>@jQ}2$eGc!u_s()P^K&4R!qDoQ0{yrl?Y%Ovk$Sku`5A7*2w?eWtDVjG-7y zio`}H(qoY$f)a$^sPILGUy+Y1!YQMMe2s=&Jbd&qMr@%JWyz6i&wuu@0nUmn z9sz{kTjJZvcL<+*HU^t$53H%q9=KuI5ZW14FsNHv_Hy+ZbhNnxv z0aq|2(}k(Iiu8Jqe`f+mG{k1#vU+t2xhS^MRysopXRzzU*P0h3;dJuRuYR58p$(k{ zQqB*b)wsfCI`C~chB2FZn_;e})6<%a&<`lREa>knDu-GcFz9+92=CSxjRiW@!R9~7 z*Mw8RsL@CfHT38yBFUt3U2n9{C@^4S*$~DMJ3YvZ%ony= zazk~uJeHVq1(!HXckiI>x1^SrPK7>84Yw^EvjnrU2EbgFVT_8LKJA{QjeJ&1gs^EI zMzxz3gT=NOemDG#J8B~?8`X!twuzy{WNDA_G+Z#?qNyU~AE;sk&p);dwCx?-<1B9L zpaf98N~sz~zo91TXa*Y7+~u@3qOMn*fN(p)N!-G>QcpkwP&Jd-ymgK1B;A~9k8Ga( zDvn}k6nnJx``n+BgNW*+7;rAkRLmcRnP$BY`BJsgxi<_|_vA$hivZ#N2|bZk zebQ4|hH9cW-WlIS{+h65Bpc$tig499>Eg98QxU%;>5rqkrs|pMKUz5a7|!uyBSC_LPeusYMgyg%8KZhvW1p zA_#LIDBSo5;51qBmvP*7agEKeFCJcLuxpfCs`PmCb4x>UP!H^Va?Trb@Z|7f48ee0 zc{tq67B4^eV@G2QydoOHo^EwmDOa(mZ$T&*$KdtbU(C2?nUT*(O5Nj3p)C&2N))5G z0Rq6;E;Jz`$;9stU;h?@+r@WI@#l)Po2CmmY1;Fh3xmNoc8f(9aV$8$AIC#wFxyv5 zsf}*l&S>IpwwVHlE#~HRy2`d2P#oH)Tz&+vsF;trL9IgSW}r3F5E=Wf7N28=z#8&5 z79)msg&f!U!~H)1VL+b0A-GaR>N9l(r`{Ehj?jx4sfnp;4WABYQivr6q@_)>y4DAv zwNYqwpp-+3R;WAj)`Zh)RCg7b;k{P?zXuaAzY8L=aoty zo0XVs6n?O>!Km0-v!^Iru%FOVx2}0Md*xy2w@|Ve=xsBWb7zgbsbEPoLlEd>3`WHK z+>~fr6ppdn-JI79P}ED-A1_$XyTKB=#+TjFSh6Qa2YEhx{DhOa+)=P+UQH4~tLwtu1tuC7HfT}FZ^@nR47)p4d zQ0YZYip@ABiwLvV>D5VKgJ1jZ;fc! zrr{c;*=7CA7dY*5^93d#=&UH5Mno}YPC&5grzhvb`*$4Y%R!cBjJbSSy=J5wjLA6B zy@qbfXM#&Q(3IM1Ch`#b}F z{_HrD4ZHqlPuWyKIXcC;!%gG?O!s`U!ttTQhxA6&=oV$3^MRciCoyn0RqdMsueD{9 zi!`TB`X#YdYU0$X9P+IMq8(bloLQsVBhP^J%;ZfX4}&7NV(f#!t)}$O?Kw8^`22H9 zprwDPKr5i0r|(}cEU~eI4PK$OaywpfLHml0Y~!gS@HZ*#>9Pv^s14&Z!u$`QjVC-m%YJyO{BTzq!D zWbANry-fLZt~~W%2JbFw>gHI{o57c+&aQcEw|$ql0wY`K?65Y-I@EB(tloNiiz$&V zdrFHwuH;mCfRs~QlQv0sjDwZM^A7yLZ*WZtWYPxGGbH2i7bjc~R|+sf6>z#^)05;S z&cX@g(g;C)Od~L5ydgEzA`&5d7hFdQ(G~%7DC131#J- zk{OeZdp^OCcRf&qXe(B7!7i0k4!HdNx37n9zWxSlc$s5CZaFC3GX=Fg+u-lg-EAY( z>ZbSEEPFT{A9-vr6B-o68moLhak1Vm?OgfVCoj~Wqc}X|V4R9Y2G$wv^XDA*#v67% zI$W>>LoTYrVi3GB3V_~hc)E3j0@j+LV_th7&o}fjphwqb9h1q&=H8R64;F>$x~*Wf zPqjq^G|KE@pu@EEshoRO(wZKO=qesu$--%?fq~}kUtAU+4_oj zw=II_X156c^IH*W^iq1yDW{9f60G2o5kP(N z7_f8XOe|g&im{AZt7|OOlC4~Rn&JH9iZiC^CeK*1VcfXKLhuHQha0>?db`bL2Kq?5 zao6Y;Oj2%9&Rdk{hOyhyNd-qPI7{v`Y2W?0f$j~PQ1!L8H572~AD^S(4D2ns?_MbgaBJ8d+AV+i(2kZc^0MB_ zr6jI3?II!Z5hv^fLW|X9DRi=gdFUG6gRE5)#75Q$Sl6jg5vT#`wU^&zl-AM(w+Ki> zpwX(e{L;u$Et4A3Sg3*NUm(VcnbYZ-vvRczVqA!o3lFBX#h%vaO+N)Dqkz& z@(7K;e?$X2IT4dhrC2G$ZXj^>yUTmQpFU%oKbGc(yo*iF2xM7j$(v3$j2UmSn0_=d z^mxK?&s$4(D2pd?sT?ZkHVfdsK!3{Dnr=B@lObhJ_wuFoTWHQnaD(hEih0UL0~Py> zvF1K|91owc^n#z84!&GoaVQ{74@GkC;4a6C_}&R-9UY?Xg z!qtLZSdmSMvW!STb7>%kMQ=(nL9FyKK_GuC@Wd)9>0=ix&1rXR)B^D3qCGKjz03+q z(T4I*AS+h)k}1JT+~A5!%MdmN#RnSGX+w7xeB%u^Io=_iIr2&g%~{dz_+c$9g+PM| z6=+fxGsW-sS@#)XSxUkhc-#Q{m4SGZ|ct)lQ+sq2t)UT(U%<8&{+=JXyK=e*2* zfwgjd&~vQ!QRH3Vwo$5np!ywK^WVR_V%LwGDtKdXpQRS+q6J-2))va=VeRH^p4Ab- zH%xf^7Rrnby9+*9b_Z=YK8){sHxZ=`seiRYlNFN7jwF45y^U7cq#~Z^@Oom}FuDCw z`AzpF7Mj85Ci}1zJDxlQDr%P}CPqUpieN~EFcoK@ha1@_#F$nvf<;|Jqwc~&p8^IG zz!iI=5CUAqIV6bdsDOtJZds#qiNq%< zTQlAPPFKg_NLZ`)uQ{8BRqkUx7j}Vi`YEr)j02OHfiBo`b$aG2p`0#)5_q8cv-v)B zm@)fI7;ETk*%_}^>5{SDHC8m|7h*kUz}$iC%z4SnRIW?DU{?I>$x9|3PKMYENEh!r zqc)$1CAkZGi2T*i#N-A{T|bRt{{?@I0|R^|nio>#|i?f10yRy5dfn>W?DU zx#&pKc21s`_H%)0<>b5n68uO-NS+pw0!ULCB(V1E*hGGP?MsO6ri__ZbV5Wj>&`nz zQXQ`6RzhW{Au?tf@XUWv1ke3*M#^{)%i$zGS7xlu5HTG^16LXS^6CL zk`L|t@F%|*mY*CCzr1u7U&YsT+B#iOf57i@Yhb-CGSvJ7sEYexiR)=)C^{KC(J6e^ z2}^l2|07uBIabJu7f_H5_lN$%$Z)b#5tXnLYFz;wkqU8sF0~l!u)RqA(5ZZdI(F4& z@z+*qDkhEY466fn!!)LYV4rOG@x#A*c{KcYKiwOyetX8}yHE<~Y@=vf92MftBHRUI zv^o6)TLgx4Eai)z|9m+A=4&>@{%c;6-{ft#1C~?v>CU&XqK?%v7DPF+1)sv59c#Jl zHhoro#3t0)Q&v=18iFL}&CsRt=G=R%*gIZY;!}yk^W$g3oJr2_zy5l7&Ww968xW|A zb!hc>?34v-(){+a-pUuL3>-;K$Q7G{s=e;0BW1A!NRA*qc{s~^{E753AM6DB^;HcA%tjfTaH-%L(Imr zz$#R~k&fTfO>1MRCW#~w#;lz{KhS!LRIRz>-8%GAr0v<2X7Ed|RRF_FuPh)KL zpI$y2{@bH>!*AZ)4*&4`Z?HC;P3ZQ2N5c;8=|-jL2@Thnx6gk13yydBe0ax)bsRC< zp?i$nHdb?!3CbZS^LrNcKI??LSSEMf_&NY;2wldAbe)GR!(834&5Hekdz)9o)w%D6 zU@Or^yY6S7@Tw{FKI661AAa&;Sa6hwFRyR0h2Kez;=fT0 z{`_3$rWz8;?6zZkw^=n>H=i&3~nq!e+rcCy|{BHjW4=xG&6yEWX(`S>;W z$AnZM*Jztre5Any81uQYTh=!QGJ4AV3`I%<9!rtE#o1<35`eFdxhKBXGJXV!EN+cSt73$Yf+8BLo$_YzIDT&F6R_^GCHpS zxdh@Pxw~)9hByDu?}qKa{wZDLQ!Etc)X=s1>8{NgpOnDdxCHX)3qK2cIDEsx^({w) z96(ZU`cJaj9Pdz<_cCupUWZ+Z%!|_}P&Q*L;)_OX~rhJJ5GRqO)#(jPbcI?z3I% ziceAOp_nsh&Cjv+pVQ}4t{Wv-Rt$UuG?i?Zf8;An3#5dUGKLZ$UkLlIkGe6dAU_Ic z#zT;ftIv;})td3DaG9Q(W#t7O=h2R}_;au<^Yy;0DPZTHkcweSDu-{E$p6ZC0WLCQ4AxT(`JI*7#!HPU#*7<_5 zm`_@F>B4+G_KfcmKXGe3PfCuDXv)QYK4!Cf#S#uI-JsB(n&lIWV|Hu*@Us_u;07h$ zaq|Q6;<207Cx?&vY-eLJ@4@-B*al}VF20S>d zC7VsQWI0T5$IlMGw2JM+p-0d8RKyQH+aLb%mv4qSH^uRbPlj`z+T5Y6_D;^ty-Huv z*AhrVL~NUtT&EMqYN%riE;HS&LOrD_Ld=?UM1A@D0KCx-toZ;4GvZpIM#qLA z%<>>B9!wsvAX=p}PP_lfkAFnt^I9Y~iZmQ+?Vy-0wRk$d2d4XC)N;vAY!1mJ8LPxh znC{eW{BsT|I-(mD#$C>RjB2;+WJP{luF=}L{^+|P!~JV!)4W!Ia_+hjf*0DgqWKyC zO1&Oo#r7GQvE5|?{)=I7~c)Ko?AT16vGQ5+;$l3iR=UzNkGe)d`+i#;ZIcDv{t^0+w6fyU}9n4$ZkTD9RRNw@bED z`FQP^v7Bz9!qJT#^6qXvm_x(zjTh2TRwpknfyaz}lgW z>Q+z>FL$T4_kf34GQb-c>MC1+u$?w+u~vuyyMvm|rE zhr7S}?J=LSJ!YYsSv@ybwy9+}Q!U``!^HC8}myEdtaZ5G;G)6jB6dMf*q?DBoERp+NO>W225{!8SHMv zN!|*8(t^_QCS)7kMx+rFUi$9#zm!KN9?Q91r_YUf7R>WpmBzmX<3dYXlJ03-?w0Xf zL57%ENvs`T|I|XMoEW|=AZxjLcOIEhuDrWTXFJ=xP(BXhlnIs3`!}DGg}~YAIXiVI zpS+oy4}6DHtLeDXM~}Wy2c}Bw?)B`@CgGNGps#*riHI-3->?DV=;=!)BrKf6tYh}P zqhe)7d4+}WX6j?EiEPZW?CuPY_jONfeq>yn3RS5=Ac#Mu^GE`83rIR;2?;2~5T0On z^C}~iOku4&6AV7AgxqvG)4a9H*rLJ(K+2(%w6u1GLfqT}lSd#;poGrT1hZoSSC-K0 zHD%3E!uuC=xUTfrN~Y_n3$tH z&Um$Ye(u4sn@U|TJGtQi%Zox;p2v!~Wi1ayb*J~m*?VphX5KumTR?j$bBsN2E|miHEU|)8rXyl)Qi<3`39SN; z$#0eW$Azjr!LhK$OQ+Cv!g%qy0;naLR#`nO9oM|^bVQTx~_Nav1}+rwpgKFcFi9D zwgZgEi(Fmu5k@R*KXpN6bu;S?OGOS3?m0H%eoiy63BpgEd2o{lbLKAdPD5b6n197W z`VGIu9fzj#ZR8DTyEEJaH5V67U>p|J>?8vwEB~l~%vibg*eX4t@67D5tq@?Vtt#Ey zU@X7qHI&MecopFH2SKLNR)XCgs!b_X5vg%s!Hc3?^4QLXS#!w^ZkAe#ejvIuuhU6Z%(`8p@3 zn?8HS<21jpuF~++3VOVO<1%NaS(2imH(mN+Q-Mn0F}_Dwp*_ZizOa1q_GGxd^aXmh zXtBuc%S+dElUh8VrZ{nt2hYcLGPCV&X%sx?R8!KN@!Y0+&)`Gl^Ekh8LPY>b@o~8qL4%R`OI}!d6bY$PSc2HW95x zfqmfD=(XVjCunF}mPA)YCOz#fEog^7`9fydPgqQtAge`yIU4aIYM|584}n4|#==La zSHT@P=>^~G(dnlc@_Tv;DB>iN+USR;gu#43w~=|reqo3@aI6+O@nYp4@ zdY*;hye1ShE)BgisLbOU-N4I3$4|JyOgzaHtSk`%uIB3AnKUM`J1p8u#(*xXobkEY zOU74su0jJIt4_dWlZ@R$!g9$&f#xjHT(X5rMReluEjwWT_V4~~IN`0lU;N_FIB1hS zgq(-S2#*D6MtgM4fA_cl$Y(6}hX3@}|B*#!#)>HXUAjw`WXjT6g*x9xS=3}iJ)}^0 zs!7#St4Go)R`7&xFQ1%qoC&*mj(OMRZst#!Xl?Cn*IYGt&GsB)*sZCEwrho}8*ZkQG+U}G zNe_?lk!sqy71PQns=-YOk_vp$9AlF*3_j*et}+|#@KnkbzOjVY-0d%3e9A)gpADzy z-{$M?ckCa`(-j2;jEmHL+@>%cHn@yq_o@QAfz=bRzW&{p!&^Sc^ZLuLSo?bwtLo(B zluu5a@RHQ8{`p^rU;pht@aYGv?rYxK1J~|d;pgNth`SX_E9}}|{mHx5kTI&0rIxQU zUZP}gcv0&0+xNpczZZv3hfmq`aKmN@Par!*DbH9k@@BKXkq6iOB`)nAZ;~ZWVt&HM zsSu@?M90(>T;Ln*cX<=k55VNZ--8SWDJpE)j_Q^=AfVCY(p?oWj7NJLnGiHw+*?3k zQb>a_Y9sCc!OW}2ImJiEXg5uwBV1zA5V6?;GHqPTWJQIE3F9XohaK`b z?k7L|@$k#vzd&J74!Yr8CJDa#>&VL$V#inx_A_B2-|p6#v|Xex7NrY^?vH==^Wo3` zlJCu7<>n~N9opjVX*6hf`RQlF|M)-uFD&<9In%K02c!#jT~QgRfDHN~bB74ZEK6PC z0ma8EraCNSTO*%!zdLz1oV;VZ*ZwDb{O1@2e;Zz}U%nVlSnIssI*26PURPpPhDloa zD-9d78bs;`Qx9N^Kqn(ow#iNCbo{Ar%TgtvM$9ogDP)d$0^+Cv#EG zIkGRu9}a)@mwz+-@WqdZ7i_%U;3Nj$B7Dz;{w-gh@&$jbruF`los%Mvt0gP6M(f{1 ze4SpEQPiz_?I%p!rl^uKmdcG`+Qc^ARy%2A*B>w#` z9jp7_AM=lbwBFuw$juP5W37R^KuAWgF4LqMS+Y@DRk5*<;8mkp(~35xk4r@@&9UpcgMS*q5rmlH+RfRGzUCv@Uovz33m(56v&7;+ zUem!wFA2vgI=zexSSYqTXSmN=p-YVHs)3jb)LO`U6ccSrck0VU3pTIbV*zI@NN*po zvl^?p&F5ls2BBTNUAZWkzjCF#yauXFr0ArCaETd3Vm4lj)nU;RheN(!vc(f0K4ixb zf)iOe-tNnjo#CG^2>bFo%TH)`_?J$}t**)_ir!M!WtLnDQkEsN%C}W@D;gDlaGkaR zpw-PrBMOkTqDmb@DnK$+w857*EFsoF#S=lIQ9@yoP{8ArfX=+we`{QDTuY8UUtdSQ zidfZB$U;8uDyLD>1FywWOfKH-mKitZwm4a6VY0;)mhSWoTJZN5i{TTtJ{|BFj+HxL zP%3^ZZCCWRuRFb(E3GD;JL5WSq^#+pU~%SV0b#5b3()(Yo4dPr1cLha(_i# z=B$Ipc35d?MGH%Ap&!*S{Ua_}kL6d3MVJOxq#L2eYszh~$yw2oi!v5gL;;&;y>SM? zZAzG!;ILb|9H4%&5p)(7-!Q+PjxV7QG_8dL8-WPg#~X#(2o%T%*Hwk&NujQ;L}lDm zFVY{~DqJD1zzBWMQ%BSPH#hsH0&o$)x zc0BFy{gaFbu{Nk#X1h6m25jF^@iZ+AE?qL4tWZ9g%WfG7^W%O;v4P=+@CSIh?y$~0 zCGrDCtDI_aam<=t#APC^-?uE7|L**N@WJpN-rlq1bi?CF2NW|VMCt2O5vf{tDU~El zF}jMG@<%vH>Ef?NQoqW4{3=UXiyRB@0qIn@wsP69{vM-~%1DS@e?@g2Qe2Vi()?9C z3S)U8A)qb@6&$a{RDl*x1=qpBX>>-EG0^E2to*7}8DgLq7aW)If4^fU%X-`My%`^< z*&Lo=y_^WxE%w{b>TNKt%Ra!2PTbe067js4v7V3gZaF%{FSEPp!NZb#8_wSjznq0F zRc0&>53>EA66|ufn80Kk7h^|XEwyfr43wLT*<4LYm7|UZG0B%Ka`Tnu;g_F&G2Cu4 z24o@J6UW@_pyhPFq>BW&NF_17Dkh#V>#hTknzqAN03UPmIm!DD$i~bBc*4`F)S@<- z9amP}l~s{|q+klKND*ISW|G&cI?^bl;zay0US-KwRjVY`@jAj9Sl#0l66y%(blr{#_iV)Z=2$`&u6mBq~|h?n>sII(0%xaWt0 zaQ-G^J0L1CIpX31RKX9gDob}gtlyCkg~6#y7MFl`oB~+EHY5|02pj^n6V=vTvtD<* z$xB5{CSKpaADC;h<=!!hcyyDF<*XNtYQ3Gw`eANoxpTJufBD;g9p17)z2H=ZInR6V zQ2=*KXSe~*l1F*&pgDYbJUspUIZHe2a6yT+icF}1b8N&|)5YZ!>5Q8|Q$($#CynoNlYGl*#T3SRGkg~n@;Y#| zg_wlvhb62)SGKXtBy(wg-5f=zVAF9VTcFOI5H+?ODNUF&_p$T|NMqLWI?YwCiL2Nc z0#JK5Rx~h{MT#>FyFnDnpBCqa$6-G1y1{zTGe-%W!lwI9BTLJ*XP;`YfXy@KuU~&N z{O*^(CY2-No`1^w#jF=%?T(mnzvFB3p3t!EtDYVQ_bd+T!5$||C)tUANUHeSyM!u&-Y-Bt*<6USs=W>dJs@H^4(r3MrxOV6PQ4E&sj95l$G(7q6E=v?C_?Fy6~vWj+kG8cAI0oiK5a&CqFMW_+8( zlMqht*ciHC8|3%)=&1Yguz%mYQLq+ywjX^~vz-fBp^U82ZH|-Wg_MaQyj; zyrA@qqdc~r@<@+&hxb`XK6%H}jbC#h<~Qd&#=9AwfA(znV`jk1<`qALk?&~59Ru-eyJ&>wCm|cmyUHM`o!!KHLFr4(ZLX&i z4YN_?o>_EO!ZTg=vu#!+$KGpP0~sr~;Y2VxfHL-IYO~3xD(04z+ee-r$Tso22;><%&_u51mJz)VwgQhRh=j5)QH{^hOOb#z>r$;SvB0X zxx_lQWG%%!%SbWCOh}>`-F2*WzgSxV_uP&ZKZ!If6}QW~!OC5^h0rG=Y*yXWLbmah z6M}n|JucaTrK0Zh`0ZzZ{+Gk?vmdcd3u>fYv6`i9ZJ1h-363=90l!6=<|FvvXZ6eG0>;wVB{z^Us9=P7< zCQHe9{1#s;NqjaN zN=UQ=RbH`MiYH4W`+$|2S6x5okJJSpK|18Jfv)Ki z&pl_5gE4cvGFD{e1f{_gf$mF29;WHrbU*myB@5l33>VL_YAB~i#$9t-*WNQ`%pTe1 zBRXfmSkOE6E4q31n4Q(mw&lUi6Z?E^^o|#tZrDA&yU#pZ3rJq10Yk79{#u~q4@l8S zW-*?+7=O%F?WYR(7?dPF(5We)%oM4}F3}z@2B*-EhD+qu3+$2JNTk(|TmWmGu$4G! zC}cIiAFJ=Epc2PvU3!|uD1;B4i7U8O{)(~i+X^iqRbJ<-mr=M6`CBh^geBSTEy8d# zB1v@PhNlqkuG#47j%U{0+|1x?*w>zXtL>I<*v{_-&zTSSRvy{I`8N9sInSE6jW{mk zDwb}+R;?wA)IR+5;9qxX?``c5FWB!l^Gp)D`rRXT(J-XDyyMJlXuBK7-Ph@gsi>TH zrEn2EZ`r}eCbIB-=+pp!+^W&Astp;VD=IB&9oNM%!c}0}4p8ldix&BVEPkg!S`SV$ zMj1b#ZnU$|nloslMeVd3(X11$N)z22sx+2b@fFNeF@z^aV1Hzl*O5CbLKU8&qBV_I zs@J5hE=3!jM&Sg)xMZr#5wDk#OtY~w9i~|u-<0z%nbWwLoUkkT2usE{$1s4I7Sy{R zbk}~kW{-`qzUW?C@Arpup0&?!7M$w9iT=!H8OqU(V>-h#Z`<*L)Q+1OSn%Fpm$lun z3b@H^ddXPymiM3Yauqlp8=@@gxwsWV?vV1P)@u5)m6p(^@fTa_MA)hq>^!%NpXQkI%Gl z5b!0Xd%9}SIB5Y3=qiej2+=oBe)EGmTHHAnPfJ*{Wh^l1{mD#}ka6cBFEsi3sV_`< zYMAwK%A!akCHI~h_b zE!z#Xo+PV+ZZO;T&?a|8r$)-sA4w+HVs#LT&8QH$DZt`GTy&VrB_)zDvKpZj_A1S|b@;=s6D_(` zoN30(`-4|`@p-BUGC3AZZOF1VreUa39Uql#@S^$E<;n2XKmF72A7A`z`0m`pkHXf_yrYtzt;o^)RhyX5RG6 z*+U3ryI`#}WYS2Upx~|@&&k=^<3@nb8{S)f^@=5=&p%`10W!@Lz{30U}*6QjHJLW-A`N0!}uK|d5m$Hxzx8?x*$ zEsw@@%8IL|284&Jkq?#HKv`!?W~isHhMQNv8|J+JxyObAO_?*&9cHsR)5UH%ip`|O zXS!*06pR_i_6{>!mRWXC5FapEl1hZJBcDSznSuD>oa`TDnI@#PgeZ1iD?st=u0_@A zdE=|Ze#&eMMLg!^de1n7D_*{53+~sy9)3CeAH#n*crk1rbN-Cgi}HJuD1XbZIukV| zvNKd|q%B|NueX;Fk&fWyMohzdjr8zW(j7eEk&*%a{2qgZl*i@QvR&vHQ$8*GUBm z@K1;hpTJrcB3`F~crgQ-(G29`SsctV=RB zwagnV;0>D`jyRM0fTfvDZkWsgDIrY%?vt zMnfrzQi+Xx!v|gxx&T#hypk6{5^VTH8slblB2h4rIno(R+vyiN^GJbPIL3UZEJ7+o)T%H%VHAp#Ed2Sq6 zGWOFN)+U5|cHeL$F;;euJ%c_rv_82ZyaCW!&ITLa{PtJu%=sykC|*%OIg;8Ebl`r-g7du8)K%fs*)2FjCXn+NhqvSx@%}2x)5lI|H!{qzgjW|>Pi7eh>{@=u?&wo?MEh862kYjGOFnwbfkV)ATUgEw zgb$m%9=#DJV!LNeQXYNIXFa%^^Hc?B>wkkazUJBU9afOus2?_Qq8RZE>_B)i@a`w-{511%bW9@ zaHZlbdHr#Qb<>g!e3t9}_6%#t6AYGWY`aaK4SV2_TdUr)wdxZ1BhGNq;(6nF+}aY$ zrjmHdf=7P%ViQM5=h%%~y6gonxz9OKbz{@5UerNN!y-9m?58of?2lYbsf1SFa*q8r37cbm1we^}jyGW1%Q%uT?2>I` z8+>Wto*mb**Xg#DGv!DATx{qxwz7v(Wk{W@-Zn}<2+fy9#ivRul&(WQ7EAs|bRw3P zyp^xwD=&Ox*_03xmakQeD?iq(B^7~!3!{?)724DZTP$~aMoaao#mp@syB4<*>)5Ra zR)u#vZqD8f*Qak#JjQKwt6TV+jPV?d@1P8BfA^@~92Ti{l6R(!zPgl~%%u<%<{Zm- zixN8ta9e%0Y0-3bDi=-2Ynty)PS`hym4$@MBQK7RhSzj^=e%U@$8olt)yhBgwoqxiI>LE%bFE3a1Jqw;KexXAV}zo+aImigTkO23B!vN0CR=h9HqqR>-W zyh--3lFR!Gp2VQc3>TJ+XLp`F!;(TDrQgt`MxJ=MQeJ#;)jj@=c%8UX#J~oi^3(gY zwn`5+K{FW8P+5f`Z!WV? zoHyFoP_T)GvXTz&JwGPvb|?fkt}KHrz=MY#UGnLO3(g1JXEVYv$A3h~)H(Hr*G;zM zzrs>#ou0VafOkjyPRI*RRDH(*<$kHtgU~(T+@&8UKJ?O2EUpt*g7s!_hqmu9yP61h zDB%Cw)p<6zksM)o5lKlWDx7r=SFZ9G`;-6w-*Mf!BryR*AV?5_@bkPq%cHyuyE{9f z)AaQ8B-|{R4;wl{2lCKw=?V1u0^PaQb4sLwoh z@AxCxi1Wi2;++E*3BlJ)1g*FR90tp1q5MGzD~4afQ@{Uk_2J%=I`H|N_wcq#dY!C~ zYgqr>3$@w2r0??~p1kAy=W3f@+ziRJ3KoSvvCqM#5vNcK8udv{1aPtULP0W06JTEI zRMT6tDbGbhXt(2b)g2z06#B400N3n+wq!{r0!atr47jv+PBfD{CK>F}W>{u4o!V<2 z@5=NkIux02hk(R3gNqvc@DRov*czB^;5Lg4gMDG=eOJbbZ~6QRoiR&6FF8QxdbaG& zfBBFywRO(vynMW10>N?Ft?yS^|KSWA{d%G$is6Mg`wV_r9FKO6oeM08dWPiej69;< zU42)Z7IDM`vv59MT;%96WCfVbhd*D)uP*6dTGox5M)(}xU>iSt{ZBW)x$XY)&wqFC zzWj|*8`huDiV+W6?=+Gie2xNy{;h{YxY~-&#b+H8sp&f^Fk4 zO)k$0(&HRKqxZVHqXV(RuJgD|wok)Pc62vXn%r|-6&X!X%c0*+^aK?ioyppAJ}ZO@ zgw3m)5V?JZdjRT|v8fZ69g|X@_&aAtv(2iARBiOJIt){ekhoc{yKk4X?h8vb-!S7P zKjcTZrVc@*yd{;x4hBWQaBRKgWd$yoQFzp@_Mmsf_q=4ALl5@W7S#T9ZeUJ-XJqAq z0~O!nj&b%*WRCpQ3S|{a*(%KQSKN@(sbh{ipP|uolt0oDn~?^BvbWAB%nD}w*!>2O zf%Y&$=^;rOE+kh-`2uU>(jV`y0}@&Rfy)~Ga{VX^i4{dtaTNI1*AH)eljey74#BTQ zMiWxg6qZ!M9vTbtxfSa34+Ye>zQ8cHy;Keo)*3|<1(6PP92BtqX4Q3f|Lr;swHcua zhcHK4KhvCe?AB3R$$bN<@wL{2&-&l^lc=7>YP7bH2L;lYO*?0XKr@f_j@w2iPj z?6yHuK4B+byerKaO#?20m%B_yifOJ72;oilufPAfJ9GM!c+;GQz6jGhY%(`EzgMUo zy2F0V;&#d{`-Z&eXjnPC@1UvivqmJaJ0k`kAWi=}<`r^kSk6Ovi_=-Bbj%3h0WbH; zF6PX2$~?B5``?&>gvOXfTzGCHUQb{*5)sd(OuDxf^(@C@sVZI=PYNnD-Nv zc}kfhjx1vw2M&QRR}=d@`xYj&x1vq|2|84g1oU+4(`e6o$#ZF|Ht#RvzP`jw`!Azl z(8*^nZR)W-xpI1ASD)lMdd}7BuPS z&mLhTDP@C${yT(uk3H%B&{H&zPGmQq^a@`5lThwkeWW@2?RwFDyIZlD5uF6$!cE|! z=ZyxVrgA6_``9p9bVXxXPfU{8*J{VAr_1!?1bRd=982cAS}$9Koh3)aB0W!4-_B* zK-+`O#7~%6X~>^uIL_aEb2I`D0FE~Sm(ix*pna`0ag%_LH7>3m1XT3A#JRNIxchc9 z>wae|;EzACyS^Ko^TneitWY|NJ=xPzcHxrmMsKzQg=E+7Jk z>4?f0fjDP^!y5$itSu`rQD)OLsBC_*PRNA)2k~Ft%+Z82t2uhirV%)7vR#n>ghSpr zjZe^r6Y?^baUdM-4hvJ9RwmuHv^vio1f+oSqkU~fcAnx<;bU>uTTT&ud+`~0M1wkI z42>&3xk51U;7VBgkmkQplgK46e)F&7BAK*r@xJu&UNU#$k%R^iZn^4aCOjEVhS7jV z@DIfDje1r+0N&6r!voDNns0Z1LzDZpds^IeQ_q7lHrn=prqWmz(4J|jl}$M8eL9)* z2!X~eo}CP+BqW)~7XqSz%p+-|5$iceeIX!5_Iq|mWdli%h_F280lIQ+qVu_35m3@N zq_?HqZ{M%FzkgwXjJ3oQ+J5a#=^4j)pitS69-GUHDv@uET8-W|!)Qh~vLp{RuoGaP zo0C4v9QBNmh%dn-aG+*vk)HAWsabc7qTI0M$j$AHk~}95lLDs85j#Mc>BF=5J+QcH zOf!7O@`3P??{#WCEdu!-+3Wh5IGPFR;K2{?Y9Sa$OTYm`AAjjifBi@I?(?r`)eqoc zvyb4ZU4=7r5bOS-(NSUQH6V{smn{yd65lA|TK|2iH-r3^Q$C%=YqQ%laS$m>#dIc6 zjdCYWR5rrl$Oodqoly&Vx%(9^6=tRk=yVJoX5BdmI%<*81?S-@Pu30qj!OkXi)1tu zgmiNs7x?IMsb<)s%PxvzS%Of$+A$ZvN3f_AQzi_kQPq&*ZN7G56x5ml^M|%_9-+;B z1X)cXBgxR+POt73-4**xzh_|+{H$iVN&{@QUOv_AtuNPM(@_}#CP8lMT7PsOP1Ng^VhN%V)iBZ#1BVkN$BT7|kfyA?U zvdImj=>(iuIi{OsfB=r|<>JN|)A=|-yrH%dKRX0UUtUx{LSjQg<`F!FCWH6@p|D6u zwV+TA3G?b5$`MwO@E+j#CS<}JW^m_~ZwPwK3p+Nf1I8$~3DEcE<@Y@TZ#v5{4{`x_ zns|h0!6xZnujbtsHcmQ&0V8_&%9dA}67lCp3#3w3F8Qy3l4m5-_>>vJh2d&K^U;*h z=V!B?7KjwuXX0-j_UhGETeSb};sFx@7ZaOf4%=E6qluq%P$0cqLA3+7v^sm}eSXln zP8F#v1CGM!YC?CGHNEAfOyZCQd0>TVE(}m8vrCq`!cVr)8`B{hz2O8Zrly^=8`&38 zGa;2Y@+9xbqX}TqRcfiMtSrz2ykbhhQ-zU#2IU^UDuh|TKu(BWzL|w_ikO7?EhT}t zi^LJYCw9O;Y`%9RraryVy~6#OX-FF9M}%n%VKm?s!fZrtc+|fqugfif1VE=QsR@e- z7A(_k1|^DwEHUx_|6lP78YpbJwL9ruxGoWFer?iwqHDJqT^B}wsnHHHl=5~A(Ou71 zbOsjLC%T$P;uASENh09)!cIf=@KA)U+0s=>bj^gGz#!etK^UA8<}M8e))~{n$+JLF zfZKpP(eu_8yu6tsh%6E1?7D#!0s#pBz`;rt;cbwG;n3VL>6Y0!BQ)f&9nINDOI%*TM*>3JiN3gwVx2eduGKdQpYMy}>p z3c|ORzoAtqPvS^;|IXb<-`NI4p&%f2T+%@lvN28gtMN%77@byRDq zAca*x6`H1%@rM`dwRgf0aLrsLSZ68Y6~)bnx6e)?WV_{&{!RAX=|&$@ zmYlLwi>!;xihZGN;ma4v#LaSw2;Qw0#!v{kiH!9}yU#VViNS;bdx2vX_lzIzy0@#Z zOtM_FzYpr2CV9tzyfa#Y6qz5jL&(Rv0Ilc|H(L(~Coq02wh(-T-|Hp{#PrVDwBQB#Q*-`&(w935?m8%ObjIc9Af zE0oRb1DfDw{n%Yx-=#PC0jILsQXxvGg1D3h1B*_PH0To-ja(2Ge|S`rTCrXP(z5uI z2YZad5eo*oe8ega>7xNu!OLPEIN@aUotbDR#VWrJ>r#a^UkbCfAYU(vEEAzCXui z{O~J&RkfoXLUQ#FBBmL(rLYJ!Fn~JwqbeZpC=@GR-qFIXn?@-SMDCVpUJS2glOZ(W zz}_CV-(kr~Qr!j+|t_ByyMTsI8WHQGbl1fK?lq49J=hnO%7IXwWd8p45l zjcWl)9&k^=jlI*t`)~Pq)?aEa7c|9f`X%E^Iyqs zc>ycGVA?H_h%0vBa=cj~(5Va-M3wo*w~AFdwQfR%pX^=)3tTIX^eau+Yt;BFmD1#a zIl$uhq!z}9=-X&Ga z5={W)64X9j0$N!7x0WSspo@I4GK-U998F7Q=r~pMIK≥KPY?Ri-~N+(~C{zy|MS zJKD3R2*tv}{DUX*QSGZME`NT8PZ_q08^Kgu0v}>o(|zZ^BO5z-LDd&XYW2J`^I*ff00s0C%1zJ!fuk>;EBq}K~$jnAK? z6t?zldKxX{q=Nm_6mv032?f5vs9D|Q*Kr^Ry7%@F=ITN*#3 z1?><-J)5iQlq<9W?%uR8C-afU=``}(<7EXDA3+w+{tC^>LoMo+IKY_3+eP+#qX0nk z*-AvYR@VGd76FJ^H@;_<6Jzt+GdeK%7&`>fW_!X=DDHS@t*K)FIj;Eu275T z*o4*=i5G2{M7%ek^r0O$7^pX2yCIrz$cRUdXSLT$v#9MSvi3w3Fwc#%3OLwqb80@S z`xDRU?NW|{6h|(Eqf{vyLPdgtXob}$zP0ZHooDhh4-+M2yZg<&l26hILlNJyYs7@+ zI@6z0G6bzkr<9!Ru;`~lb35d>a(lPMk`0WQxx?W&*~CbSa!J%4mXz6G%rk^-7$vTf z7CSL%WkyeLcukmE@Q(BIuIUiXm(OTK%FsuoxuAL`Z*!DbqGtupaBI-z8}I3Hr&LV< zDm@ATR=+fZ1xUedMf8W<#3TYyI7ozqgr}+?T}H|;^&72h8uCOSHBHK8grGC4eFxw7 zYF=DKQx5-`vV+->COiEE!_61DR!OM+UiX~Gb`EqMXt-M0>_wIK5*Z3-g2CqM_ijp0 z>4=BEdbA3Wy0$cdwKV=Jjcc?>#(jajw@<8vHf3=g$Pf@wi)Mcp=ujF;; zY^$fw(WW{*a}|if7?_0{xhRRW$t?94LRl0BCgi{sGwV)^%Wg{NV^7abf;z8H3D=#F z<`IAt%6Q-#LV;U`$!uS^Bu*UB0YJl=a`~z@(gF%X6*d0_@Q2ISq~#8U`g~ufdySmW z5|YT=sbpysAv!P}`_zaSLJZ(T0;J7|YUtDMC5{=QBDs$W11=mPtVw z;NNfep@5_|2Tp0K&|(8|m$tdU*C*}g4f|hT&o6HS7}-FAra5YLf^qtDD$l;2ICWwFh|fTx%g3L z%Jr+~G2<*pcJ`hh$W*!~*6WPe;X;9&B9K}3dSs?J+wuvFos|CpAz_T1j~IQ;00000 LNkvXXu0mjfgWQ@b literal 0 HcmV?d00001 diff --git a/example/gluon/lipnet/asset/s2_bbbf7p_001.png b/example/gluon/lipnet/asset/s2_bbbf7p_001.png new file mode 100644 index 0000000000000000000000000000000000000000..2a7e269f14ded255242233e9fd9153220a8be2c1 GIT binary patch literal 36768 zcmZ^}18`=+w=Wty`QnLf+qP|+lYFr;v2EM7olI;^Y)xz@Z~o`pckijX@9nDI-FvO| zTkP7qt9wT%DM%v1;lP1_fFMdsiK+bS3;s2K7^r{uF!RDQ5D>UVYf(`pX;D!kB`2VT z^-psU5UGfiG-x%|HO$c2p0gOt@L$PuybG{IJ)tB$kup+X!Kk7lLdd-PFwu}0DD}Uh zRnYNLNKA#r8W1DlwB}%0%F7Ro20|yg`blTH+qS*WKW%feftMFev)d=zO)I@1@#J&? zMEI%TL_vAv2=|Zqqr<=F9Z5mJn25mW;!PXG!Jgyeg??Gte|2pGh_=J+v*btLp1;4V z4!QMEa6zab_QYbta{_Lp&Gby*gSA2O?@+oF6jF^c0YueQVyI#-hisIycSqI=*+&J~ zdErco!K0x4N!oPAEFi^<;W@d}3KIMA%gC*}&K^F&W#^8@W%XPzX+%FW6B>LnH&8g_ zK4h~G`$puFf15QVmhkCLM&B6lB@pGLy!#W3errf{de{T-&Ei#q{fx(|nSAZ7C&7!S z6!p;gM)_uF=VmOpzvOg*rKWm3Q$dZ++4ixU{FjxiJ|4Nxx4byN6EF z$v=A2xab%3I~micV=~fEQTRvKy356#>15ymFINZDE$}183)xA*0t+~-V*V9TqsecU zC1Nkqwr0MwF|?5h$NhU=p0bx8QWbGTlcG3=@dPA~C#CN)mxUJ&XX|u_=I^~6VqO9j zbl>sZPyziv<^l+Bz>LR!g2&S|wbM41COzvhjA7g*sy3cpja}|7WcajpP;X5PE`a_z z>*9JZ-&wG@!V>Fx+e+ny0U<>Xc)4y3_XcqxdQYV3+dy~(%_jqyGo8eQZ?d#IJtIVL z8Zr+cQ3Ayo>oS9x`h`&suF#L@0q&2CKq5q^1^0&t(hP)n4D{6qp9*5c2=@;pO~0Q5 z@=`$2E|(Q}*?^}5tR6V0u%HJBnGt$7$iXgudN2_Tv@#J*F_^W8S3Im4Bs0;EFw+mIgk(w@EQ~N2 z5m@CaW$|(rHN;BT<$%A!TN2VrjLLa)T5K5dl34{gbAG2%Y`8fQ1~Cl6n)&B*HK+Cu zb`MNmc%O)(1LekYOtDxL6i`Xg;{)g>6HH-D^yyLQep6J({FqTC#)$P#T7p06I54_U zbpyAe?0&nMyw{WdfyF~A3uGPOJ_vNA)5EMqZi9A*a7S{7=Zt(0r`%(|)$kzn!OM*^ zjAI<)+@0MA?&a-M!a70l2a*z}_Y)tYwxZg=;s@gfD-V(>=UNH5P)H!FMx2Kq4O8rk z?<3#RM~E#@aL0>C^DXG8aGznFu~?@+L{tlLP)JsI2j=v85u4=%bRz}WKD>wH&-^;l-tDq z6bDwRUR(yd$-9|8bw0s9sXqlFh)ci} z1v{>R9_;Sb|K4jJqLkzvr5inn_2393grpnbK*-7z2vE}&8IB(*pik|9(ioqS9Cr0;bg8NNc6Oq>F6 zrr(mxEPJ2!X7Ocfl0CIfYnze@1s#qoBQ~RpWi$kIP?SWfS!^k{B%FZFU#;vvscGwj%93?HX;@UCZ6-5unb=P`w97C#SvBHfAO> zkiE#(cXg$Pv{qL4`P?h&vE+QmE_{1VIp3hZ)yS;q`uA^<3=!kKU5YaKcYmSp{`Ug+ zrpIIctUe6K$XZNY^fU~AYU4D$&-uI2JHoP7GP|=an>N=Sjth>j)_}{}jdPhJ*?SpD zH;7BCE$JQY=KHN$^RC+DwdBhk`Azz5dTIb2aHputawC@!gBs&Ag&<{G-9|mDO5oz? zqNion&T!H1@pI%|kZ>2jUH4LVOQ*3uh8y?i}44@+>FP=#O3=k(n8%UD`p zOW(c6*Vi{lXJ|#DU3rtsjqGb|y2wCsBWo(NGRt~;WVE08w&6sO@`KNTZ`WfL_IE^S zk{4x_PC31JjgqbXxreSyRxy8T&vo*5MJ-ZUY^vduKevz0xyuoD39Ifb>MUk<4o{l9 zeWB{y>z&WtYeKELK7e6ZuYD1JeaoA9cmGIMAdTUn{W-hq!sqn?W0bb6mZi2;zo~1* zYwO8tvaYqS^fkZoXE8isPVUWym&~T{WED{Sx*;s9uL)nDeoCIeUJA$iTZt>{PfR({uTg?6AidvZFfNa@_W+nemq{J~_IoM$ z7rEJVGZ>XwzqB++H+S>{Yy*fA5KKsqZ-aIgCiC*z66S+4_p;2=3|a_{CBy}6H!7Oc z7L+O7UL7UZK7wQu!qbt9Ww3W zzO$+9%lWdB03XJ~T@VIh@B9wjkE$!NbnspEL*vp9jysp`E#lF_DMePkUz`4}OyWlHmC_ z{tuavgy_FSTx|JCwB(hDM1fA`MC=S~3``^fa708zd`@N-JSt)m{|o-_iJ!#E#l?Y# zkSx=`H!Lhef_te<{sAn$CADC{|f70fQ+{|8d*{~)@;{#NC^}i2|4Y(;QX#<1$M}EI{+B!-Kl1k9Mfor4zp4np@iG4I3Jbt# zQq^;VfCzy|iwUcFfM)k-$Jz5bm5yeQ@?Wz8aAZpCJ9O>Z<|T0J-&ZuDd=7RF`W6KT ze*qDOl)xoKRYX%|)JtfVxm{=0Os}%b7#C;np*ng~I-V;Mr>21PELaFoK@EXJBpn)G zZ9oN}rY7gco0o0XIqipz?oNN-x{mI+zwXFlP~ctHQ7vmV?sJF#ZuC2!!uKX(wj-fq zDD=6}k2WEDgx%;aVuz^FyszLN!N7htyguUtDfnHB2CLPmb2m4y%$uj*Ixuv< zwnTO*f-&bHzFUVdIU~?Cnqh+gEIwmGEBR<<4!dB@~!_{Up$@Xl7Bv?UVw04rR8()vOndyh2tup2~ zv{9d{t^n4n=L1AscW_Q{$`lzP@ubo_JeErq8m#_}5RpP0HJ0FzGP|EYi6qgL#94$H z&)@<(@n1m$ZRZ{R6k|RYXrv4&gz}YfAkA!{LA%&ssmE84W=OK@ z$3mr3&A#LmEUY#nLpQRPAwESMLQ)FW*Bhd8$Q;OXxyUJ(>TceyGQ*eOl(avA;4BbU zlh1idnL7l9n+W2e3?0$oFGV0RJvpJw9-0qs6h*cSCEXEwFs*_u4SBur5Zn)Yo-<+F z5T*eWy$T9^IJ}6+szF3^%pZSQ*K!CGV7=q|(51)` zvlEvIeZ4x+6(`#F1I-3uPH+tn8^QtXA|YD0BG5>{=^t-SFj{uL?{cC zkVZQdOHth2)byP5mC~%c^@>A1o+1Ap6xrbudhNOT2F&b!iw;BgI1+M2vVA6l67lWr z&I2<#uOOISYivnn>nbBw2-_V$(33IJzH)q1{?@78Q?YOf(*f@aCDa!Nq!Y?zSoF22 z&t!HUdai^n)U|hq-60(8vp$s?N}|iYq6oe!>>qGP$^U{el5Z3Wq&Zj2OL50B6Iv%O z&M6J?&mLL~bjr#w1Pz8WR~sS?iu3ccXnqh6sK5kkPO zTc8TIZ$FL_%zUc$)xLvPtBnHmbmavuWOn5BCoiiIZ_(zSuuW+Nf+;aNo$-4?QR!qN zW^PC3`{C43EIm^}5w)7Mi(N4MCh08ONv4vGX_URs)2b#i2k56O$2&E+ls$O#ni1^Vz>xc0DpWQJ6Yq ztwS190@{Ernnoueb8A6)?%fBDn)6DQXFgwy;Xr_nK4F+zy3!*F+{d=B>V#E%d@{Ql zR;7+Y3dH7!Q)_|_FYV>dtEM9X4xpVLXO*&ZgHrh&$;#4Q@Ns64ulxW%1l=u0Ej>{z zw}9xe1s&a`fVb7Id3yCouoHn-??`y&Psg_#)c7KcHF6!m7C@r}eJHWXLg4QYSGA^x zib|;g5}av|qQs395}OZ7As1N8%=dk64OKV!{uxd)vMb-c-=!%v=ezm1N#D@kx2l~% zdUh?zfQ{&Lg=SdfBp_7u!k&PNg#>yKnG6W1T~fE@OM!@Eg`+8o{;hIe222-wg7xM4 z@ExOPg@A?_0)HZvzKX|g2j?2Rf7j>1Y;?-J!yP-05ftWoW0eYLuRi9w-NyoA2Z;_| z5!Nd)quOw2yK?@-S!DThQ}(^hi15&SJ5)YU{A2>@q9sf}uHuKEG*K?L5_cwvAMBzs z#Xzj=a#-fWza9J~3u74(iq?pLNcAj|bITWXZg7u1bKJ~XNUX&yz%^kX3HwI3rdG;F zt^=7rcoj6UUI?jn{ zG;3E@e=v9dDhiXzNZ|8cHTd>}x0QE{`g**$H zGYy-F&vPIqEX2SVN`)$yS>^b1EKE;PYAMB(|$mAPhy4>N;=2d$Emo$9hv-EB8 z=#H9!71j;M=HZX=J7w-tSQXU2dG!*Fo!>Ao3=4e8a*GVnsK*S=lL+1er!MxdIu+H<|vlv%= z_LW#nX(nn=qE3laIizE@Pkd;vpl_g#D3^W<36tf&a`m|B zJuzNPVRFl~6T7il+HOvr?u@n0%!6(oP~p2X*)s)&h&(|@+#c6< zSX5eIvR+W3+sWq=QE;Zi-b!`@>2}W_T#hM^JjDhS#~QPTYqx`!-ETx-Q7RJeC_~Yi zwUgBzglL>i6zfr|(X?4T^HjlMpYhCF=g+gVT1Ql^x_d~_|4nhGmH2}QraGCV_`@t1 z2=#Gs5bsE>Js@Z3Y@YlJCdE}eJ6DosSnqBTMroxf*0LyLauV=4L5y?>ZWbfkYM$Iui1l`uI zj*MO?!N-28guTK(C@GRy|Jd{$!t~$(61=@W`D!H>X`P(DPt6_shGH&Sk(uJW4#?`n#~co|-fO5UfJ z#dY;^k}T#qhEkJ#c*{7@%a` zopQk*XsTlrf6O@U1LTU|)X@yosro>7RQst)vQf3i6JhSL*^FLqw``o#>Lg~ld3Z^6p10nfHnny>Gv96gT&+sG+}}^vUZZDos>#vU zZ*gwaQd-$?l}PISL)ra%HtgoN0I`cEkfB|_<=i&Y)d*L&YqzMsf=t$VHX-*gDa(^u zU%a$B^;`fEep_F=+T7}8yGv18;PL{t<5{C=?S^yPbZfhZj@}W`wG%_ z_s^)Hw#(ZudvAA-x!PHVbvWO?q@W_Vo=@Km-A~#AC!`GU0q_l4SpMO)KqYG^Tm|(&ZdT|UxXN{z9JY* z>q3Km&`zTv+tH_=H~5FiaN!uh;I!LC-u(c?-)3BLQJfyg6hKTQFz_;?VQldaB|H?H z{``{4VYt@hIw*`Y=t4atoZ%mrP*F`2QWl4MfkVq9o*Tv3zAo>zcE)R%qmqb8tC*{n z#RDt_#XtaH<908rwXbWDL*HIJsY`sXb_VnEv}>nzl+b@^W~aB*diy8sdz{itPivE6 z5!}qoD>aSl@{=}idu*-DQO;SE9G#a71cR*dSU z<#BHCXlE_MX2{D?cFe!~ABFtir%`U6e%rmTSM1>E3k!7D7XnssPlT<`)q9cN>MJwr zuOr$Un>(X{3xZ^q6C=`@hD-S8Z3`UFPa8;ml-^mF4;E$mfqjzqUvGy9kfr_|aABFw zxt&L9(+hu>AnI@XMu`FJ97T>wu+%(On&wZVpA0Aoe$}D)W zySn+rVH-w^z3-3n`8~g^H@t3s*&isppY9d8wRkQXbidVZ{S+AUd4&WoXpIrS^henu zBg$-(J6GDLG5RsMrK^T%V26~Vx!M0*OLCm|0;(DQ^%Y?+Ln-i3TD(BO22FqP`g)26 zJJ_GwPJ7eemnX1dQ;bqk2A_Lq;X+}~MvE+wtk~S0@h6Kmr6(<3?6^;Nq@aBv?P%DO zR7tjT>B3x$x@DyvP3X)%J&ooC4z6D+rFR`l$n_K_r7Ip01rO#5? zI@c!mICsVBszsZabAtWIvSjo3n!0QZyy29@@<`Sbf#gL6i{v;(yi;VFy`6mZF7<h94Tew(*Ah7K5TeJP}O!D@TbT11f0&J=bpyXZ*&mn+@{`t|dcA5*ozM8$LSa#;6!tRzQAsV3bvHvA`E zK*dXMcoF0Xn3S}C0-H4%4fFp1H-aPq>*JT^yu94jJAUt3PY>MSm^YXYR)#<6e_vf+rhJ^N0y$wA2^UW5h&%yy|Q%;TiU1z@t0aJX374OD-LJ`n_ zp;keejiLeDb?1)f8gH_DCj9m1QFG70J_7;Y&nn?GMyYUvLPMdL+rGXLt4J4?&7n1^oWflB?k>|WF0o|e{S z4#Ep#_6EOQgS$nMx_NS~d@Xp1HG+@&e^UUAK*4%Yv$ejs&an0AVY2mkUeoR##W{Y8 zc{@-wrDjg$0$@mGC_V%{>DgsWqVv69!f9fP4jZ@0 z07o&RgbcY`Rwu#W4huaSculs|o`6UyhSLl&Xq+2Y{8cE+;1PW<;71~V27VHl2#HCC zF>{9T`XGrXcE>^HK|>6ejskh_z!r0!CTVy6xBk+w*8=RV-1mD3xbeK>E2`=<-~0CW z{Anc5cPrM~!vD3(a3N05wu=gt-mFyPVkmq5$H!OKH9lnG=Sa9n(7?9qg_{8A;@QOhhAkP-tYyM^ge=hK#a@ z>-*uC>Lug=T6!xCEt`LJR7Fc0qDTjD#*jsCNqY<8d#vCV9jGKnUZa;~OJEg_9~iXX z)f6{*G$2R}+^VgMu$oAl!)6UhekA9PQE7W1njcc{9B~E(ROL_BAwO0HZ)bRvN^f@Z zSnFQ*+5v!Z8x!M`Pj|cGyxcyLtesnK*2iwxw+!hbj&R4fL!)ddoR?(a@hMLaXQOgC zdJBqX4PY|$MrPhVQ|J>D?Cx1M)e9IanBwZ7tS`T=x`tLS(|yact{ntXq}+(A|PO$7Jc6w<60FbBn6qFY%rIR<-Uis%@gB zify!0#?`=OwP}l}$!4YyA3!6?u>y{I4to#)J4!-BvL(9595+{+#>A0u4&35s2-wRh zIMMOQlN4xZeW5Wgzp^HUvxA|xKgIYz-W@_=lKkUEx%sfG^Kw-N(@_JgV;g?JxZ!>y zEz)~a&Ho+Ol^D6U%GLEAUK)GClrIJ{VExpaV|0G-o(aBN<6epR%GCIKmspAyb$F({ z!HY){Vt&U|Q8YvwMUcDydjByt9A^1C{9&)lw?ku?zuCp+vm`b+U&fN^mJL{FW9hGUkH4-%F2S+{0xg1hmo!(ruffpXiVy_025KIxCY^7Lwi) zV8?jZuIt0^uac|GQqLT6L(0RtL>|xHAtA{xW^cQs>HX0Mc7XElQUB(_r8xoi_HpzR zj>kbfl1MJJx(W(SOc~F$(ibVfs<})(6ddb_^pO*a_BmHJMqoB7z7{f=Y`x|_iyv9H zHzB$VY{C3^zu9Zmz9flhfHy7BaorKk^O|MB+r679J9;{UwbuayIe`WzmrT02o+Lg% z!bsr=B7+Z`T^7+Unv4$gL$e%xd%=;bVU%Q~+X9M!uM4=FIsj_KpHL$5J7F=?Tg(pc&%=7qu z;!fT77!|&4%?rO(rdev4m%6T4*#kYo_-3?l0TwPTyOeUgj6)jN22tQvZyg z&`we2cE3oX$6IIlR`dAE*EIPKS^b1%lrL)3=Bd=bh1&&fp{Rx*I;gF_RK+E5#m8Y~ z88}H~(;H^^FaVS;Luzf$wHLKdzTl+01Pb@1X(xDmXWE0et73bH281y$OdUW+yffXS zQH*x{7}VdxP_TB6?^l22nZF|B3sIWg1f77=I1GS?VgOi$8a7aY&<+uqIpg9`&AcXP zTw%$!_tVT%nyGjT?_B-8qnN%e0jhd5?Tdt}Q$*HbPjQ)SZ&?gxjlN_Cgvj5{%y8Ul zDSpi;)M1IU_JDnL1-WBG0t<)J3eHc{G()te)fe>df1pDq>$n?#KtM0#+807HTJC>xm~9wOS3_E z0}7x;M{ULwN4`*e22ht{yhE<^&SN5SKCnHm2zQn9QF0H8qw(W6pg&0US7RW$eo_#t z7hLG!Zbh6fPLk`zzK7R|4L|ov0cbrmh3nqB_^fe988?RRMXO1|3{6_}1X4tC8VD-% zpzxl~4f_0oLPVvTRp_Mj(Ezqyj#niWp;w-qN1nW@x|6yk=pFro4Gdq9iuChNvqqmMp|r%4v5(tiK zx%bXg^|E;m;O4GE`$WdR=55Rlzr8vf0y;$Hl7q7FcmoqJnM#@h?&>V=^hTQ4OQgdr zpLY!!)aZOi?9CNDL)t99Q#iS7yOT=KVUe&w`aECdH%#XiF%^lWb{4|sBTSBFDQbx2 zS+RX>HZI1LGr02|d7##8=CfVHaZ_sk9r#}Qga;(aKtws%xr3Cb2#b>8J!64cqfxt9 z3y-wc8B{FbJ+p07m`MLQBOh!}NcvTnLp6V_40XI0VcigerC}SY&L#0AH%`ZNEq&56 z^tSba^5fWyi76R5;7x&WF#&FSAh4#1?L6?ceYIPL*A&$w%zUAYZHU0ID!0Wm=BBL6?aLq_HV-5u17h+Zo$3=MSn{xX~`- z4n3wyH=zTgA(UI5DASs{9R4>~Ip{5;GdGWEcZqN39|jd7o!~lxm`#Oxt851}6CAX5 zIDo{WE7}+J0rwds&k_X0*HnL%6ui-#-9z*V{&r#WFk0~d(`4nTe6K7p$Tqp8!H8#K z$xOKfa{4s7zsZ>L9_DFkO`e1C^eXrT_m9MPdv?RKr}T>e9Ffq7n7r`Z`eXzmJnj5n zl!f>}0*+NrE>9S=i$%S!=sN^mk4*L-#6GX%O|4!&TYg`LUmQsbvw~gri1Tvx`YPot-+G&igc0}t~zM#oeF<> z%Vx6>19t#B_H*?4(NFkw84d;o*X51&z{+4+ZG?gneb?gY{Z&?vj^en=WgScD?{zOk zpX4Xio4g2rPzF@*KAkCH0woD`&vX%>p8h( zbW+9|1(0#pfih-4_b8=BLUhqZ5oiZ}sqp3TS`^IdgVDZmG%Rc#qetFr@xPu<*vv<@ zQgwA$G=zi7SoCfTUJ@;wab+KfR7nIXo+j&q72usT6E=BG$k)ocEW}LKR-#2S>Z}`E z#5kWGHQLtKFq1Yd<)HVhN{VjmLv;*AiIv3`FNJ)-$@NcSriOB$FOer(C#-x2O3UHb zo$3E=i^z6Q|P^C00;_TrjFzwy>7Z$0vs|MW(Fe zxr>BU^MsTX9OB*`<6>|+HpOCqbmy-NG&o&>h|tMHS=|;DmHpFV!3A{sELtzwVrssh za^AxR@lij%=e};U`fN7nrr%D_Na?ohy~PP{!%IEjL%1d(?%|Rq;;aNX>lL1E)mXMq zL`-ww3=hs~p64834BM*B8amwc3qJYFY}FRn9n{xl-%NpVSko-}MON8Q9laKfkm9a~ zT~MEP=$`QKop8O*ezEodY!|$y{Kxd;kCVQy(w+lQN4tG;KvLcNg(;mq<7pDZdQ@B8 z`gYOD<*ntv9kzzThNa=MJ!}&Zk_3Z60g)a-u60oodd2354JnG!GZ#@%NQjpHYr~}8 z#XuEhEVW3$S(=5K7D;vW)!Gxnlv#U+VY9Ycx^^Nq^&5YC5i*2?<*L=e3*yQ#B)UfA zpM>@pJ$M7+Pgq<==C3Bn;D|j)jML*Hc^;dM=*YOfS8d(T%h`2;dVTa^aj3HH(kAQ& z`SGKF>(dd4nMdD`3r}>lBOSfBFU*qCJ5Hs{>|;e$TwAH!KfDvWW2uq>phE-`)#S~6 zCk83-yBtOE)!DvdkoAa|;%4A{)GfifGuihE8Ss)nGqN=L$1PUksh&n4+IULeJB`H`Z?E-AwGVM+y@+oR(t69Q3oQsEi zyWY>(77xzL%r=BksxuznTK0*f@YPl4Rjq7Es?jt6Q6ItY|9Us@akDYUk5WDQM_Z48EFMftSSRz%DsCUU}2|&WORQtA;>{^Gs?2 zx-vTfdkouB1^b|`-%gCrrFgd#Cz?A2Ry1t)sabS3TB8d)FAl%1!6KW`33Jr2KpxM8vfiu$(qFoOeMZ0fo<^cYReGVtRD2YL0iVZ;<1yKZ;rGE0{)xeP`owfw z0!8r~6=i$FdvG6uCT!+=nj_aW_H-Teg0H#=i(cBcJBA^wWqF-ytffCDt*aHZbQ~7u)n)Rc zWNjm=E~v15fG$Z{KD%-AyYb!V^^(j}i?wN}0))xz8o@0jgCe$Zr~o`_($pG{s_G*x zm(klNH-_q*UlrmW^0w+P9_mz`d?XjvjSftCh^9P-!-k&G~}FojDm8PF1ah zoZ}{*1~CExR%MN5cm}?Bw8SU7RTnV3S*j`d;1KfB6Fo?ItxtZ}Hqcc?<7qaqtWFi= z)Pw9UKld36#Aw=1z1FZ=Ir!Zqio-_htV6hO61n{dGK_<;Jf*xUxcf%7U`)Nh z%^_-c_5QfI8HR0wTaw@#Y?W4S zF`?(Qz^*s%< zSi|bZGor8COuLIO#o4wg2>j#e(k`h2W>E|81oay4PF3H0)tL+ye`7Veb-II5_)|2D zg^3z|aV;blQ@C(^fvGP+f95X9gQRRDIp^Pmj^@+ef1EhOB0Ufql8Pw1V`1f;9eQjI z3XR^SyV9?;Fx5FYH5E3Ea__Odm~A&BdCrEcEUQj++rDI;EcCf_8~J_5_~*yR#~Nv- z!!d|&N$9*Yg6m~@+N!O~>7T|3jHauzCNg*$51UwJ4??vLmYL=&Y65bnj^5iNtHR3- z(^ovmAuEc$oFSL`!hP!jel{nJs_vQmBPfEWbfxc?wV>8n@4uuRGi9EnHA#Pw(pOGN z->zO^&OJu53Ld9~uYyM_!C0%pOe$B$98IeKcwgIe(n!O$4!Y07aV%}Jd@5M>>Jv$0 z3)G_|eZh(UV$lgj{`D<``D0{Z5iRvWJ!BBKl;h8SvZcrmCFuyQm8s#jj}v?vFnk6K zI%?FnsxQM4B&}q|dnm0 zAHtPcQUK;Za5isrOH6VMw0|-=$`i&GSRJF?=kTEW9R9$5G?{pQF)B=z6IUSxCNKz_ zg}XW_Br?PmnLtZu(r~jh7^VX&;!hA|Y@052$p=L+?X$h!9O`@@E*!nz&wpHwOA@k% zh071Cg8-B;%zQ&e1bQ;`-CUSf?iZ!{M(|cLuAzi%hl$NU#J9Y`7?}l-=tN}rqu23b@SMy#Jlj1m$lqIP*}b>)T0CGJ3Us}4 zG{oj+ni)Fj!%u{-?PdI=!qKPE?Ngi{BQZ8M*`4^CEND!slHQLucW9dAKzhYie|D#6 zQ4z)vmFsD?dQZcRxg60*;74b~WK2zh3bqcw?{L*I25L+mE1Vvvr9oLFEbIi zB2TxEeGah||8+x*?@Pl56gmZTHd+K6O zaH{3rl@qCO4S|X|ewJpzRY2nD7sN*;>W*t^I82R#&L9@;%F`XQ9y1#@N|tOnC4y_L z%liY*H0bz#ggNEju252X47kwb$jX2a7WF1;VN|Y~A3Mvm_U;!=hHF$S04U@eo2D6coc*R z*_6`TKhI4bYGfkLqj@*kC5Thtfm5n$HHBE#$BC3LRQZ4V+Pc<|Vw?A91V0T*o!x*> z8_#{8^w{&9hw6uc6=f;%5<3nyoTT09%FhSVUe}-}Cf!*3X?#@#G}&)p1P8g3te$+v;%s2qap$vY!tBnA1ns+Kzae_v z*Qy*PR9i>XU}US)7j9bgkeE$+<&Vb0G5maS6Q!+_=T`0PJycj033OhStJ(ZB;2IA5 zEBTyYIdJ~w3u7obN{o=I98VM@sLHu-%WqER$xZ!_j}`yX2u>Lt`hBxt@Mi4ml99+exa2}wm#_I?oq zN#bLeAW*hTbepRLXto>ONQ(RlaVeIhO<&+m#t5gGIa%@i2jJPXA(`uCm=b?Tgosqw zdhX&2-o)Hr;Y)cf3!C^WE&Ru~_t1qE`5g`A;py#msz@5@&Wa1t~>0}5POQIIRi5e( z-J6sCtpC9CESU{-cKgblR@l9FvkE#miH>tk1#mIxFZ+v#bh7$r9*gdS7rlhlS?&Ic zWs~))2r_pda(m0Mk*j5i40d|H6PC3?&CB+26FBQoitnmb6V)2#+qaCS63S>A$6=V- zruBfp=MlHv)k!KMG)W{8wNLDlTLY@=Y4bXvvhy)%=8tQ|{ay<8wHdBahxJGCe%e#ZG2Y z<+`IC+RQ;)D(5lg$N%>(IIBQL@=@cGEY`LgVyP45Sh{94uEFBi)jap;xAJ>_wX}=s zSrFjJcoH6nV!mIT5UY{RoZa<>phLG2UYYtA;i?o+S`b+>&t1|zlKRZ(_ z57MeL`@&#h?U{541E))dlWjz|9d>Vo@O_eaGt-B7RH&ty6v;4(AY4gIaO^bUc;Lx7 z3>UOY@s&)8O@+u8S<$G4$!Lz2mHeucimy!WlYXz8+^Q}ngP2Au3is946yEOGNA0l=nBRg@+^4VYm^A!vL>W>m=?PORwK`H7SS3e+YM3h)pt5N)o%;=@ zHQeBoZ`@JwzR56+$!gmsryn>v=KZmyuu1f%UxiYjU3R2#+QqGzKx07>b&ssFXpgWo zUQt#Jw3VO53|h+c0|9#X=YRxuva(6O>_3`F9<5{T2 zNZLR1z?J>cUnNccgrsj}Tir@de-5XlVsF_Pes%qtie~=IrVMLzbIz$-jb;n!NLztu zVRl#^kt?sm*e`-k(}A?k|(|FAK7}5`pv{1?PatHzL{fXRVQ5D`|H?9ahJc?OU zRGeFA0{*M!p%+?1e#&YRAD-@nb-wU)FR}qnh&2(~Lr&YZ#Xhg)dkk3ko0vZ5~Pqiw80>dN54D)MjEtfu$WRp}|jt zpb@SVVQ}zOvC`;-D0TP^sX-53@XAI02rN`WnqVdf+v4257}FnV{@_a`LMV3NeQ)gA zJy*1xc~B)2+ivIdcY6Uf6^uXapgCC2YM9@qTVvALw9j&yYC{!L? zK@e{J>gW@yNLJuT4HSPx2Pc<=$0f>XKvIQSN`b!<4p=&JB+3jv(u8m>T~Zy*_*m7X zAyir9p$M@NmVS?{G)s!|rsX+9+jf|!KBI4M)9LH73|N!|@xEKwL<6x*N*yUYa5XmIbSfNAqgZ z^r0fGPRJXTs!~k+CZXXFnDU?UG!I$j7Fl;oiQ3UoXI_vqrFU+gQa+S|k*1Ts3efag z7q?&@ynWYRa$3US!Ep}BJZFub$AUO&))H-@gnMiac*uIg2ORI~1|?U?+DC-KeD4)d7Q!e^quR3}s}X z_;rO>{zXEG;SafaqSp~8`X8duXY?)4*`&n2s=WF5>>5MvOOPNqYx2fK;ZS;%{TdTev1irENxGns}rYH=5 z3K9uL!wAO>DOH?On!6|)+g%W4*T6ytL)lo!ox{^&2)t*}`9T28|226d>v9QCOy$3M_8@2IGd4W@^ToTdbg7cYPqvBR@7+T;VF7TP&p$;?7G|@QwBV=+jI+Dd01r+BcY@q^mrfsM1U#3J{onedI@qN0PjLvhC|E> zo8iS<;arhVCh1KtD$YDcMVD;>b69A+Sx*?VAw!yeD;0mHKh%)=Ck!^f+(X*g1#!i$ zG{zT>K>Hko6W`M%SU8CwnMq>C1?LYQU~ryKl93?2n2uL;xpv$xO43nehObu0il=2k zQ-g59#`8-Q#@BfswIR!%XmCZ}?1EA7E>>xWia+O#$}3LUf5)q_QcmDgKLuef(O0~YvZ`ez*)S03ZW;Nyscjo!Uw!xe`Y(YmK{{Jg}j3oLxY zf#SM=;q{w0H0_LzY3eP7>DEc;wb@$lriB#$3@O4h~ zzksISX`%3nhVoPDShYx^YJxIHA&4LHEtxA}1b=!yKp>XEY z(;7B-!?HuMJkfP=%Q1I^^93qL-Izl z;oOX#(nV!DjU{g7Qi$(gr!1|E%K8pcc!gN(zm~BQbWdk2MsXxFO76y8x}AAI@uxsR z2%zx8Kov|pfc4XV&rS59G58K_;7xgl(v+t(rxbY@Ol>v{7EsDl-^H$MGXkXMv__Oxv?O}Vt;g*L?)u>>bSiDWn9Q0M$fhTz( zIzKh>6s2jLXI)!nnY&93p`i|qmhr-gje<0%ju^bxU-Ja4OfZ-ou-pZ}z-?|b@! zYEHQojTQQyD`gWrWSaW2KJK{@A6khUINpcfh`B$AcQ2`zyPy%$aHWy~LLKyh>pz zngvWj_WR;VA&I<3xjuck(!Sut{EdSjuwpcUD50R<$MFCFKmbWZK~&4*JxNQef+5R6 zlot#lHrRRhl26Cpa>m5blgjl z1qvvR-@e0H`vs?m zDCGbAufA+wth{S~_n$A@KfZiL--$gNj%uk;Pd|{>71q*aRxjWD(7yW9KX3c&YCHJL zziL-ltwT=Re}dJ@USRr=4{28S8BlCtA-6gFJUjJ(XYfLkyp09(Qxxv8-)2$4A^pNL z<_I?!1=Gs<*cR%^F_HUr`=JmJj3$||kmNL+e55aYL7_9n=oFztim}H9=8^Q0eD8^nArZaI9O2@l!W{B?}-Vsz7#-(Vr#u(V*}`ePJr_a`VSG(P?483#4-8HrCn zVf4&kZz!%V|C;Hh_~TTmyskREpDEC_?Nu&_*uPAB=BS@SJ;!5zd=E)!5jAw z9Gv7HV06J`db}G|Pfb@k(U{LLeN~h3Cg^UWWmsgXbS0j`g*z|Q+@DyReN);u3KKM! zHeaWU_Uh-qZWjmtxjp@F{w0dT1}HlB`z(!DS$rU5$4YHw8vP< z|N5&h+BuWiYfPK0Fv!^Q$PSdmxw_dIay;q(^F(!rsXm)r~i`QlPBcGD@e|; ze)b1{08exme(@V{Kfz*p06X0b%2I2HeN6iK64ix z;|P@C5qnIW<6FZDx!Cj}Cow$ZGp~HFi@^p564CKLX4BAygBVU+z2a4x?`G|HVRnoKfg&I47FT8~*G!KIS-* zhbW*Ag>D@sc2NVm9yM?f;&F3N*t+`F^T+M)e(`M{jLqj*Zw5TKOj4>zbA8uK;Mrdr%M+xZ* z@yh_hOvQ=|>lIdMx&{w|38rKN-?UN&V}hHh@PQMa$jT=mEEa>6VON##+%P4=E4xdS!uPn0N80C~e}q zEIimD&6wZC2bhDq-Q zul=61Q{KLu(b2rj93lANMGldN>2U=HuH2VyC`{c;rc-PYsbH+qi0L%v!QkG56QAnf zl9n;i)>Ds2`ch~{$1o=2Nm3Zn96Sim6V?kL!}&7@f8LV7;4EQR+9@874D=+EZ|Wb0 zZb#y#o9LCs_k5r*5dtFw>n^_iS}kBjFcP*o!}6`tnLnlZ@)WG6&#-tj+ir4q5x@ib zS`GqINxs^n@;zF^b7HobF5#;Ue1?KKM*4lv=)*mF%uYSOVd7g1j4t{@6O6U8nWouY z;q$e=<#2oEEnkcJ&{s4chnjw*c94(*xIQn(tlM3vJ-DBcxf;A4V$Vux1 z6W0{hNp9`D6Y@6K59t!TIpss(JRmA<7Eh$E5K@<^G8%PMvWsL3)+L?WbgB3$SSEG~ z&ON1{QLQ<+`-_fs(yQ<;PEY1MZLe$~9hgDHKqo#JFj7yzc>0T8cHfPMCvTwc9NAIr-2O@%9* zU4G@aLM~BIKb&~Q2z9}m7OkO{4j=n@X+BZLM=UyUbxRFQw65+|K_#G-l{r`{1?ma^ zIS(6dwmEs<2J4eg`0fZaUw;1@qsePPESK-Kc}5TiGSre{J++{I-^A&eD`55!vl5&6 z$xEhXJcafNZ*7iWy>7qx#n0OV2RvBSLzH!eO;B;l1aCxBkh=Eh0W$a=j*%Y(j#T~L zahy3a!RSBU!%9$8>ub2G@xcr-D@&BdtXq*Kwn{LRBh@dZQvJXSR-^~sC|J_=Cumr^ z#7UDRbG*Tv_oM^k&UBIaj;!_OCvQWUi=KiIu~b+;70LCO=IkL>!;{r4sf!Feso#lZ z>||LY*66v+%hpw^b2E#mq}FUFhI|3{iBKg=KxB#gIRmUswG2$y%-CGtSw$ zVoHa$o9+gioH)i#5z1Rc$z{l~A|f1k)Q$SCP(i#(z#1~cB~-(+q~-I4bPyWY#T&?F z01Q_?r3qq43Z8JXmPD13K;dTJ@yOKY(LQ2U)5uKWB!l<|z~uV*<%zXYrivJ(jR!HV zdjgU!l|7;+U*Y5$+RNmXZ4xKXL2^l>z=a1C0qJ-07+cj&QN|4>mF=$`ynWRU-h9+9 zzjQ@4(sEx+ewX zJRmBh2496d;bhr&$8R(D=(Lbl(!J zls3mCV<{cv+C1Mf*W)~==OR*S7AUX?gONK%94m=}!jvb2U9cZ!b?Y&6jhy-)Q)3O6 z45|`KA0odlO=Y4JOZad@ZDycEX_Rz{?-$dS127RO*iy->E|uTidM#Q)tvBA7NW_2@ zW+T*T0+U417ojp32nkJ~!dogR5i*PW5Z)bz22Qa1#YE&+Y3qK6c1QDJ8*GJFQ&4f0 zmOJ&~YGo!_+J)bQ18~NwWj}iDn}_{{A4;SNr1IPk;+9IcTgtT~Ry&BEvZX?M^1jOS zfCUHc`@KYAjt}@0EsGp3S-hGLC}Q!Bd7XC2_flMQxW$ni;ySHk+6DQkd=97ASSb^w zG%u9hSBx$flV`24O6cUX&9I87MLon?ADu9VNb|1c-Q~+u&UwCO$=LQauMag0-6BRX zsaxfde{XSCjjMl#vP1_W%RIh?+;NvF^bMOQ@<6bDq-bmh<_4S!Dh00iDFwxp&w;<5 zj<1DHZ4M%aMIr3}vZl*F3wM;j2Yw~0jF+%t;Z8n-tNV8=-d~4LX=c1gJ;fNj1iz;c zMuijwD$F>n;x6A5S+Lgcm<0(gvwE_4fMU?ZGV1imT@@g!q@iut)p1f+%JjUpmpE;<2P*+l7ITspTwf6G|XD2L^<5D ze`d@%L!FJUEsN!HC!Q;_7h7~B}^10gQ7r~8aX^*u$n>n0RDq2Dc<BMpykIZNfBB2Q zXy5+&7wy0QpZ*t)9e&Cr_X*0(L^mVVUw-p#`#=8U|K0xGf8bDd-ok9N)C$vDWuo7C zyMf9v4TOFn042arzN2aNNh2U$JC??f@$s+ks8~ zwd)C`gWS42m4m+_-br?mEd`xiYN0A)F32hVu17jmhvvtVEA#o43Xa7NMQS2=mxv*w zAW`edd-PyBiV1OrGF)XkLZt5n3?aZ4lt!foExf?OpZ6()5Wx^>qE4ry2A2I1#-T`4 zVSCme;-g1FODZT;XinL;3!1ZYK8o|~<4@XGUwz&F(_j7to0+g|yiRlVb#}~YaTsKH z*tv&jN<&NNRL%*D4y0xiYG3mFm>hmvo9Io2icQ4!G<1gDg<_Ax%SS=3y>KntZ^5>@*bq0r<`wKSYEFN+Q zaE_(F6T&ym3kU3lAM)~-{PbHW>w}w4c}@>Ofs&W;jE;lRpFaBuDqb-qLzBaMH!MKB zDyVn<CDOw|nO~A=vWG;awDkIuhMU4L=!lkmX)J&D zNVTi>^=F^4^#0HB%R3eiMT_N0T2rGASYskkLQY^{65EaD8!XRKDa74o`P7=P-&pRi z-M+#C`hlJmmS*{NrcXX%vf6jB#<*gKE+UGjlAvD67v6>uvCP600es~!Bq%$G zF+b-U-COK7)BpD0{`>X?3mJ}Hec#qT{um3t-QF-2dJ6;mQVV)A)5-# zuHbYOvG^&^l4F!Bfq>q-6z0f+v;qe3J?y)HEbo`U34fV)2Ud*NQQ?7bsd_6#4B0{o zp5X+!PERMaRZM?Tb`i`kmtl1WJ?|pug2cVUI>U*@Kd|B>;Js2*)4pIT<}H=T6kz*^ zJ;)7fxWlVEj|I$`&{TMGCy|O^3*&x`%mbn{zJ6O}0-KsfshtwWDn^-5R0lIu{0&X| zI-d_)bCE3h?_&LSKVp>)mNExpst>#~cTY&kQ43fPPMvoHuRa*bA8zt~y~^RS#$SK_ zN&CY;{L}W?laKhmG4qudjEZ+3w{MSc+7A~fEBjsS_g2A6B6Gtc(sDV2zTqppkDs&TSH>(6N)s51XDO}z^0XsIoae+yd_ma0pfjI-hs5Rgevf#Q4NtCD ztk)=X2}$GPR}Z#CZ)vux;CVoM|1}WEk0ORA6ifb}vSPpD3b!@3E#x;IjO3=+=CAd@=jhoVP@~==_1{#bTRKJpm-zQJPkDG zmb5Y$s3SuXKs$Nk4QH}u^CSvZ>IY8d`t{*udqh*Vx$~}Fk-47^ThK(ymR88uO7zV5 z^KQe&#&Ql_SGg>z(^z$uG=3Ef*h1^Z&m~=cgV@iKt+ILKhD8Y1S~Hl@I%?TeCi{dd zOc#0Fn-kE&(MxVVoZR8TrCT1floMC&LrTMG9Qv0>?^u+}ckwQn8T^}*tM+#b_8V}9 zO+<^*O@oFZ8q3!GsV2&msu$J;rs4$$m@X1Xq*hLV5%u%>zLEwBxO7ad1Sy+SeiTl) z%9xZ0G-aVIp~*~~2%zLN`K6D$9|i!lmm&y3isdoA@w3>_oWBnaz;y39GZ&c4 z@>)UQGpZA85D_rqEl1*qOi$xO4L2e8%kJ= z2^!2Tqwo6e#7lSrY5Qso9Z z53XOl+2%uqmwZ3?`@{BU=*I32?{PnN`{g|QvW8CCXuaH?8R61V%p@=a1eaX-v200S z(wy5Pbikm3t)pNYJ?$kjZ~$WY0pV(@#m|oUrY4_|;C;2rzs5IEB#g z3i({nE%<)@ar@chM|?<#&mUI2>4U^4>tL+Q= zhyVKNI{Q1=nhBnNS|$e(9wNkgX;#zlriF#DlJ!`(4J_L(*5}+wW4;!8!A5z{w%%gG z*|Tj{+2Ou8X1V?&J|oL0`I0lT>lk7cu?oKd{cDf@Kr_=Xly59`ep*U+VFD+4^5#0_ za-eiVv-WM7Bd{5Y_w0= z`1HjZD^YG}nmc{_ZMyJBz51TxGGBXg)icu}KfG-(e*GJ^fjW0s*=Wr?zRjlenze%a zyxQCQgin?|WU2-$dgcZuESa0vZDKDuhgj>WMbh$RV5oASXsRH(n1K+p>431ZRPr$= zNF4Cm?}`UQrKl#*a{71?hKK4`8tROB*FihFr)(10%cp;WO%Vgdl;JT*GetDX9XA6+ z(Tya-C?qmT`QY=u1ulinJfQ_D26E$)kb&~ly&^kl88~9Eb#i8J zvx@>cPsr=Ib3RgoU2HF2{?LB)H~+w%k=nQHfM=b)nTr(O9vz|tPB7oL<0HP*{^E!B z=;KFxIqD-;(70%p(-_zTO#rF$?uE4Zb0xM`Ey_m|OxTZkyKGElT;mYyYnJRB@>=ec z6C)hhXgqZhmvc(~JHQDb!Nd*e3!eKc@g$rCi`WcNC^CucOgxAH>)8i5>L?impETg7 zUu-vkbY1#IqsSINU9mGTF*b@Q5fX6)wHGGd9v`Te3P-~$cR)$ly{!!dX#>ke}-ekLin!!8+=`$h@L^y^O#_p*;6RtW0qe_yfsA88tnR?$tlK4Ur5ZM}8Z%hg+h?(H@X$vI~MeaoPCvoBiBeQ$gdMhrtiPCS* z6K5SN=q@Y~<6<22I%|Or{u2l*&?b-q>KOP`5vmi*R0=#epLfbT=2Ljtq-ovTp?RjM{N~rcZr^-|#bPmn^N2n;Hsc;W>>nEg zI3SZF;@-YwITzngLBs#xr(f{B;(Z1dD3Ysa=zm&?Lsq7@L^jE`#lkIC9vi(>{;U)%X_Iac5+~Fk01@JPOIP7eSNd~z7srJDs#tZc za6QfV{Wnp8%?4*65y?~DN*p!vwAnjB0Z%VEw4C`pKi^<$=uvFhx=CtRIeS!`i(~)f z>#y7APd;TWBJYHGcF#CA?)=)1UfX9w<(M*YgZd8BIyc0*4gK^PpCw!TF`MQ&I)vuf zLozQ}G`q$dd-2y%daHuxBF|efKDaN*Sw{P+##y40C;<}%D841Do~585;<3IAe#p(x zD%uX{gxzUN$bW@rY>2&2v2w3$2d<>D5T%kPy_b}y8><1#=r{o~GOzq9=Q2bcURjJ} z-m6M0ZBLh{8Zi}-6e)}w*4<&~PN3qY8Qh9X4)J<1P5}%d6s!{Bum8jvlWR^zjZ+PDZs4IMDZe z9VO;#pwRMMLU-J)GxB9OGMl3~g_WIoAQNH%>OO;tEe_tiI^?9TTaG1R4icuKC|FO= z&9*z>Tas8rCNM5D7kY$K%o#l1BGP4Ck{oz-A0eeR{Kexk zY_mP$O3@7u$fSe;_z+MI{DZGCLbjTtq)?ky(G(FYj7a+x3L#Hh#h6R?r956TEwT*Z zo2q*9#ac=9pW%YtFLmqbL#O*^^s3JSoOolGTnt=AVY#a5vhuC)3l#3n#ajE4GiNrL z6i&H~qvG8H>Lw-kgSgwyFSW0GnktD8U%o>&Zj{>OT`v7O6vttLtD&6_^fR!IemB^( zg7M2Mz1t1^>{+?Jrtj#v*(mwV0!5_R&)UWQfNAx7i<7r2jJ6%TXkk+o*1xK0IVe#o zcHV6C;*L^_J-HDVLdrr4OIfKdcVNVeF#3biscJM;CnoTYzwiU@QBg`0`VX%@LLt=a z%J_AKnLeB?fA70hW_L5F>NRt@gARpB1_>pP_`-WKAVX(a!{joZ3>doO27tt!df`lr zfS^D8-AOqMkl(0~PH&Yhh+?0+0Go zcKnyj$jABxzrrS5`tJH?MBnpDb4FL*Y%QfMyu0OhOpFSAme~?7435l4!3UAtEsVc8 zkPPGcBQ{aKgEUJZlM#kb%#v2@*if)L!dRwF_c{4lsk1Udv+|Xu{Gb%l2Do5g@Q#t^ z0rP&3S^Z+y%S}f;^iZ@kshsxDv<ZD9YfExg-k~7^a&c8psb=G7jb7iECX5 zsV$!{pK4DUFJ%P^u^5o4R42izWD6~!?jocLC&d&;sQW8c`aNBgv->53S5Ugx;5Z7@ zN|X@n$5oy(e&1CI%()6SrVhfjG^A&L3mTH~Wxx7y#>DlvOrdA3DPf`;xd$A`Te{0mw)xw?O*JE-5!3zCmE>x8|DgaPcw;( zLY5RriZXgu^(D&S$8dP7+|FrcZx~(suJ{(K>g-BZL|aDk@+<5rfD84b_4GY`o3LO) z+OGt-oL@1jgf2kv72!F%_0AYYKjFl!4chRF3l7<2iT~QZ@3}olPQ^1(<)P`W2etB2 zX80y&33Q{or8uWLAC(Z4WvYM`w1Oh95o1tiI8jq*Kzl^s<}*@uKnQeL_{=;ZfFQy+ zg|b|gDHWImTmtI_3%Y(i67F#BLwyE3J)S#Egix5b2$g)KmlP06KPo*8bVgM`(^RS< zR$=9)KC0|OM%r4x_|QTcQvRw>Xa(@j1cl>d{g>ai*T48#yZ*PIf$Jc{4~@Y&g)^T;5@yjbV-;e!Mc#u<}%YPSUL|t8q-O6O_&l< zW_d!%mA^~JSWCu9uzaP7@HMXufA%*&YY+eYSzG7u%^RkXjEF@~35&jcP1h-m=dPl4+`!FJ0@4g2iQ>AU(2J`>GC`~i z2XkgoR020t8aZWHPg`YBgrV=_Qz>&TSpe&3iA+F%zgZrlrLd)FNzx0T5P}~*^(Hog zDWAoPEh(}=r`X_G5}iV*=a~`#S0NW{96x>WEhjFp8J+E$Zp*x(3C^KJFz<;CHlJ!( z7B{4Tp>R;lBS$b6E(ud*a^x)@)s@`+%R-++(1hRl5Z$;e=4Ng-SkKlCG$)#)E} z1`-qIdzKT^MR#PCCL!#~w12pskJT*`$zEP&GzzH@qt{#po2C)BlZihnyBC-A^gB9r7wW;M!|Yos>G1g792 zPI+{f=#=mdpkyXU8BXy%eMil$gK~q|5(EFpJHTb8feVu#)8vOGQ~`(PR$>`INL7QA z*$&+)D+}iTG6RX01E!y!kO|yl>sGkcFB$24GAeDQ<2quxI3aYlCc)AM4vAv#x=+`uX2sK*L zF-SMJuMs$V$G$rzD1I*-Me4TEU*O|rgdj=}hj#qe!{JeHF==l8>pz<__8s?5zYr<` zq?wdZ@c0{=%6Bsu}h)f8p3!Un@b1N)4ZoGYhGI4h@Cq?f27Lm6b&OKRDmd{9&IP0cAT z_Tba(BUpBFq6xvx%3<9Wb^Q#!c%Toz-Pq=aisFHeBgO?i^(o>@2Wd;!nhBZ=^^7=+ zWqSGNX7=#==ELbHgqQNC?EY2-kTQRFg+zX7=^mO%XK5yGg?7QWqDGl zL}=mgTzupc|IEWnXn7K6^G;$kp#;pGI{ubfzAF$d0D;uih9p88zaV!J+M-t0$*7B# zGKrMju-4gl@s{BRm}j%p<)?s=PEFBb-fAP#y9$O50$k2-hCL9z6?;ft#(2oW^Zpi1 z*PuCi?1Vc;Bkg27hu>_NW;R6$w#Bp79@icl1aODLqam6KCp(O?u>XTi>s!JeuC6$V>6ya} zIaKEY`_G}6)&=m=rBZBGx7l`&@e703<1ucOn9duqk{Jt+ZwftSAYF!W5S(s?qr53d zKO#_2iiE%t9~!tw!2rE^*|aXufNRWwB}a`dv^$CGHUTT&FF8HH!x|NR)4V$I1oAO( z6;@mrHuL-IW<6&SE$1#LvoNIS790x`{uWQ8Et7*)-phh`lZF73gEgS)mVbtnwyJLx zWZC>?`ygzv_O*qYrwFl6Z7+ldI215eKX2X3!SBKHnSo(WSi2(3Le&D5S%4NT%!DDsQgSE-ZLR!92G>zlKlFo(wRpK@ZB$uTt62! z!=$k|z86M2jujUn)b6nsiool|v;2I2b--UyiO|&BD-pT`1ZNP|QgGMsKnqaVHcQD> z#2^x+jALK4$DYC39fPy0$pfbxT{Rmt9LFKinP7=M&6gt?Hd(s#?cBL_hgLiBzeWp6 zEL12AWi>P0p3Nrdb?QM?mmK*HItq#eJzxO&?*5TDKSsFJ*5@2}Gv{2o327HJ)k8W8 zbTzOmndM*|q8|Mvfw#1R#!@@anUAN82WW8*?appLV7dcuihb(Ydb;mg@@VTJvA0q64>a1W#0F*j-%mR_wHhcm}Dvlj|=4@Mw){3lm%(epmQ>G|rQ z5?hgu$&aiD9N}8Gv|m>y<@Kn!y1hr>7R~uc;?ijWy62j>hvear9yBu+M|2o!_KQI)5$M<RYDwn8ia_Jg}P+<@u&5xMRTEx#o{-ygFw%#??8wwLw}^ zN@L_YBs~I;`Fh-Zp%xD5{onod56uu)ARm7HuV_S;nTc^~gtn*}O&~JemceIkS@zv! z0CWf|$;gXz5t;-_i|?-j(f%dQ`}e#g-!_fk1IIG>-2L)k!APL3JkXf7qY#!v;tP4f zwCNMs1Z|FfdIg>r{+PbYB(Mlce13#zZ7REyymr`di)pi&;>mt`M<;?aKJaeJ3R1#n z&4@9$c(+%eu+!`Xdu-P_yW-Gx_+H5Y*nrhT>M+fi@<3eZrh}r5#6;66fGc*pIq-Z+ zGn>I@zNNDP0VHA{JMinxZF9}Zp2IJA=MTErmB9LeZKC#rZtm`y=?XJIiw!NN){?NU zh$#uIAI}`g;sfkghho&K`Yd#D!kj%Q65aB+FyfrUwtv8Tf@kD`dhT=#3|xSGp9ewM@6;TH5WtkPMYPSXBea4V zqP!v5N%*_`!j#y2$qVB9QNf`1+gI=qhBdvrExXayoH5r!`#DT9NHdIW#k}m=g`{Al zw*zfD7ZJE7Zv=6c3x%=uXbMx^dFE#FI6LRO5I3#6&v%Z1Yi8JFXNrxu#|(YweLr}E zm;)(=AVfg3UOrwo->z?)FPy^Uxy54N7dBe!1Oz^E>e3xP0;l-vwPWJ>Cs}p{ZPE}} zVGywNoC({Y0oSO_xb}s%;%(k83p-_%%_)ms-5_=5K|bVF9${fO_KjnUU%!=cDuBdUSMGY9ZfvsMlY*fAjHq7g%<_I&u^Ye~zmUM;(h7?06-{&5*CLD_ClThI$?D))Wmh} zlub&&fJukiXXL-7$<`CWk;lJM20IlwDgt|IfLGJ3`9o_+RJE6$wl-=h&XU`6%KKk_ zdT9RbpCAFq{I~$?AjGF))ufIPn5u@bGLmJS`&l=IwewZiG^thOAF?n$OWe zmuNY*>3Meca!*<0aMH)83`Id9f-5#I>2x!v@AiR`PuU!Ro)K{K@WOr=4EPf`qKAJ5 zu+yYt_pY#)_Y2@_KjR~N&*=jLyh{vv3#SBeNs7OMdy4%_i!Za5!pWs$kA z9qb`swmkZwi629H;SW!o$9K=Z7MfB`3R?$hWQK>NUvlWr3KND=sHTsm?xA^?I9Qi2 zGI0d$)Ftajc=AkVFBOC2Xzn$gtu?KY_))NCgzv)-zxo$ig3puzVHJk0AaFt_n+6H! zUnP-8zP2F=HfnqMP%;(z8R|#x0s)MIXNQdN$|M8{kxlMp7Dh0RH-D0@;`fn4_`UB=Dsv86qrgAySP6uJy# zlSrY;7E@`@y9*h-UuZdQ9`IGoq22U6z?0#b2rc=5zY85+@XH_#9D5w{{fyqW!ku0* z`S!rU4Oq1P(daAe)B`m523pwasC1D_>JgflN zDNJ>;zy^3x*4Jm>)pY)t$O@*W?0!pY`aX#NWPJ3gm1uSNwbsHih#tCj@FkZ21f)QL zM4D^nUiT~9cVqMQ#+S_@zj*hC2h>K0umTHL9&Z$`AP*dBR8%u@bcHJ;BP807z*ZPl z+wxuS`uc9M>Gqff3GQV)@v(rJHY1JN&O#_K#!piXqn!FOj%n1mF4;0HWh8y76augz z&h7ZQxw;#36zv6`P|=Wt)l?S97}rW9>DA)FTB7mEKjiD|iH~V^WDxn~J3lr-;^sj+ zp88I-z#tI3LL6J+&|RB#!qK>wch6{OnoSSzu~v1kEjML0PkAD@=a@WlX~=+lz9lm7 zLMwhLi@OJ%Pj`S^KR2Fg1;zO1Nhmx6V{`qtbRzesAJ`$tAw=A?F{Jf2OOOy<{%HOB zT33~!dozwgmPFuKHZv8G=MqME2=50@Aqk|$lC+l~I>^OFlzB>KR5GB`r07eJ=9OmF zRmt1wLvy^O$wfOI%qOV$8LFP|3LmzS>+KxpbC^K8PBdK@YV9Mk_eU6Jd6Pl9$b^8`EuG z0YC?3*L=n4(HExBh8XfaW;|Re6U0s6qm1A|ykwv<&_&4N4ClfL)22(A!C9iqD;K-E zDau5qQ~3M_$OxIZE}_GWihFhY*j(N+Zi-*y1DbDX6k3SD>t%Xyfr-*|&dksX7}}vS z))qzn+3DHX^g6C;rH9YXNh?II3UZ3m8Js@!R%^P@V}nwmC&hU9XF4|=nTtmE`Q!B_kQOlotY4m`eP;_snT0M6R$Cwo5ePgA$i)6n4JuOm?)YABgYZ3H z)9n7WIYDzBBVYpxu|WvccKS&1m@W^W$zECuO=iw!tguy9jNySlgT~l)#91=rp&g}$ zQ0v$wby}^PT9cT5%K)%KW??~Ah6H6x&F#B)m=69H60j5kB2y7Yj0FnE)h@N5Ksc4o z!R;9O)OHWoz8g=Q>-&f11HI%h3noU~5)Yvx%NjLMxL`zXGBezNR8oH}i*OY*g{050 ziCSLrcAAgJdZ%1aF?&`P(DaEW`10zhxqp74pN)<}bN1YpN&@Nmofa29a5Bn(-951L z0`G#5My&#tOc74Ly9vJ&XorsElkE%hP(&SYRGA0%tT-b^ojXzM(Bvz1wVbIDbTs-i9XD^BGke*%) z8g6{k>@WYj`HVGk4AU2k$-Q8YI6Ptlr2ubOGL?O`2&1c09LBIG${%2=#fllHOhqYp z9)>M4<{hCUOs0<*Vh{(PJsnf~SxsS$iapqBNf{D*wk>-fBggSLg>MFaGR|QO)s4ce zMs((+7|AyTBh!rJ;~3VI%5lR9nxAm(Hu%i220NgHVG{=_$RDJ*)FL4kf)?T6MNzVs z4;9d?#0R&O39VB&swLGR*u1GFRhg_jc1X03r_)7qb^X9p;)J0YU}Cc>W|B~ILXD>o zg=Rs-uV_M*hBvZN{t(f2u}7gNg0;5cKxovCwgApH*#QF03b$AkL=$L7uiVb0oY-M6 z?>NWhfZ??RT9y^7zI*2%ngJ`3dU(C%1^58rdM|{aYePCaDllG8Os zQNcB>%u_RZgNjT+ShFGCre64VN)$%Me@K?4WLa-W0@J#~Zq5pxtEXx6mD!uy867fu zsY6HWfhVI=0mYj%maWQHmB67i#}-$VbHKD_!B&i{y(kz)3_vVGA$_D<&*7NBLS7$EJG}rjec3>P?vHX!8NySy`N6DWL z%7)A5#KVYInwmRCT}v?4wv{^xL+D~GU+8Es$NPN8EC&P644lsNIz@q@eVSFZj)k%J zX1EJC%k3GxPZ()av7`VBp(c}yHLLf_yH%w41Y*FmQlRBr&8eoWrb{Fs5E(j1>;Sq= zTN3bB?Eu+eO0bRQmF>=&Y-xhj##Rz5hH31%N?5P_`YfUOp*}TJY_iAcoKDJZTzq&G zgk_aJ!Y%S3a7p6_!X>I7z&Us&fl<|h!dK}1(JTN5l4--L5w)Ec%osgSUYctTfPCWj zg{5Xa9CK=?n}fG;E1#;+5lRI=+LuZcK_s1Xg`}{#HGk>fB29W}Z<-PU(2h(5U7z_T zqQWLJI56}BEaNF62jH3xVuW$t%*S^eCcr%VIs0F5>dx7>M1s!9l-b&jqCx~L6yU&H z^9GjRYx6shBk2+pfp7?c1~1R{iuB*NW!Zakcijw_|FnZ3654dySkZ()^Xxb!T_-~1 zm4Zi5cy`!e!^R*rriD`o*48{khtJ8s(;~d)-C8a&91@X`8qh^d`3yJTU%mn;43g7% zAs%@+4VSa!pt9~h?2Vhh&64F7M~`RFxTd6G^n$X@aQSx68Og_(4fZ0Ps*?*5G+=dh zUC`u@og2|Id*Qn~&mziPwNy7&DDz9BfhLiKBj0C?+uc2mIYMmJj5$bg=R|>>m@u#e zjTc~2F@doKM=P?IFjAfj&tUrz=6t!r+1bg`%v&0j!+6WvUJ4vn-la(p2QK_2Ix{fx zz;f_MK_t#0FSRYmEWJ@()QZo zE+H6?{91v$?eX@(wS?AIf^>va7%E_Elas_m3Zb}Z(uij?EwBVDx-~3S!fNP_UUe#h zxLP*eq8m*^M<&mhQ(IO)ZO3D_M|{IaBMzZH{)pZ9nVA@DNSbQtB~Q|mhG@kfOouYi zPKj0k00L}DL_t(?t2Rl&s9Ts}vu(xiGj6ziLIXbEW{27nn}QM#MC{?(sSqm}Xfvwe zAHjsOGF0LzblP)S3dQ>Z0q)|!%M=QX{GT*Q2a*b1ddw|>ieL&5nFB}YHMCTDMJNmv zmN7ykBEt+jHoA4Q8LF+ERep=(|VdaQZ{s@eA3%-<_L;Z1dkzvpm}YUjJN7wetUA&3_pBq4$+K` zH}*Lklk18hN&6GZ40*Q*UlVOks?zy0%d8 zEp{2rbg08$g+aoVph)}@mJl$SHrcjzo5E`|09h2CqzALMepCol7hB=*FSx2D$j3C2 z$QWLR@h7*;htv>B%~ZFw(N)Cuj4Odrfe9d|v7G+$h}sEu+JO7uc}yjuqeZP5A<))@ zg}YY z)D@(psMOtEtnD1FprGWV(=rgXzN`>LOm!Z|5k!4Yoferzrs|BK>HMK64--+)Q#H8Ac za?Vy6dU^c&w{lO&5StW=32sL^J}Ni(O2N|595qZ2s^A72q^g>3`V*6?ib)zr`e~1s zaq4E@PBmx0kT^ezQ!Q)=x+g`C)r=degd-|9Z$edeH*o>0dD{aR5LSNbW>#J+h?GI) z_%o?4;O81Pzw(<>&R+MRaw^!eF1Zw7Fcy1d3`nBNO?&-DG4%JVY&*~yliV`#OIVQE za202uv)veM$+(&!9$*MC8T%KlJnEh1Fq7>jHroPYj77YiKTuAqJW7aw z9Ww=d_@kr$Fq8hRD?^ZN)~KB$<5OH#1`f99&`L*zj0dYCD(2G#(A*!Ey(*p;ow**b zvhG>EcJWL5iTz^zn<#`3+(S4WOnC)kHtYzS$lTD%QeT$xXvjW{f0O*B{^(-(d}lT* zpt|iRzTu25Q@$euLBCQEt(ldg@mX<#aseoC+U}n zYzfarML0}~64qCs%LJ_$@svn38p$s{sQyU{T3hOVdTRR95x^7vB=ka}f12nH;RUVO zl)DP4vj61@CKDRe5A6!q0jC_+JwpI=expg#TUag34Jv~euK#qhsHyek+Lfs zS4T@_WG2FoRFp*4sML_D;MPL_jl2+iBC#Q>pw6L@KdsA)uOgRSm^&SGB+pBl8*LoN zE}>I!I{oL!`OfK%(~s;OU8=XjOqnyDh=BniC3d72&wP|Kl9MenCNpT9=|Geqrqm3* z_CfcvBP%~X2*)5~BgP5rZT?zIyNpbRQ69q6D+CE~V>KkG#%e+If%C!eK^6G%8pXK7 zcl{Me89#~!-%ZTg_CmKRao!LXZhtrH>k4zp$8m7@lr;%qX?#UpF z^(Fc=YJY%XS7sOMnk`y-mO&^{QUNfluPJm)bj)p+c^6$J&d=bV3>h)MA-FMfXMCo3 zW(${`DR0%-WztJ!OZ6LtAEh0YBmN!RhV~_bwn4T`vrPLSeVj!vq$;lpbf&oiSm9p5 ztrxH~8O7ea+Xw9Hx>xQq@2ifWkE#rR*=O4i-RId;+9Tbo8UFmG=hG1jki``b1rLMS zP@Sl-s6tpvjXAqLSYN4Ao?Lr?nS_ZO{|kN$egg9Yb3Dr@rWGccPk+>ZYY7+A6&BjQ z@G`LFHm7;#@Qc3!)B^Kf0T&$;w?7Xyem+Y0cGYKd>$J(Y>9v_WH=q~|8qlj)ffTbx zrL-G=*V$Lt$DfET#Qf$sWSR8&IhAL^`}gVX)xP#;oqp!3VpFL7(gMRg-+bT#{E7cO zM@9F1fnBRziT&BUU?o?X!h+|_p54Vfp!_+fTH2?)d~WP`NYEqDK1EbVWJT~r_{7%Aq80NNcghOHnOPV>@F0p1Qij^-T;wUs8io8% z;Q8KE7kvw1TUmY&fiZpuUQ+(@EX1ER0xmq?n#6r1)vPi{YZY$7`?VVciPGdi#oX8OC5G4MaaF$T15TBr^fo*LfsMdQYs-c-@ ztZh|qSJFqpRcM>WZNc5DZLw9%d(I8gO78A|;=Gr45;X9h78xEnT|5X0EiXP~o>JvV z&8X2y(y3_)@hR~Mf25^!%D2py&37{~*h=fl4I=7>>OKz&RrX@Y)|(=GYs zK*Fp*0+QL1F`=Cyg@nF^dc&c!%^D1t%oCu6Wc7-KP(~1iSw_6!{6I%X{(}~XV@;e+ zxrnQZZbGn$^&!?rYwj=*D&H^fW~r0vs1=|UI3H+bDVH*a@v>*TuT#T?yX$!v+u$mEDf@{p$Cmr+gQ8-Oo<$I`E$U$0+2 zm%x{kt7a=EsJcw{*w9(#?6O?VJ#CEfrPt_gGIC<$q0nW;XMwnf!lC=5XyhBE=krRV zctw{6BX;9YABbjmqj>3GAw#cPZIjQ^uK z_`G`URB>PFR#DCy?%Z}mVN0*^cB9$~RGqq#dcLKy&bG<+36s@js~BXnmdAnr3I9Ef zB5gw3UOT%|>}>z6vuVo7WG?9beem@&JHAQKm*0>64B<)%hbQWsxivr(@n;|2?gk%dhQdK z@)%6vOcCVd3TOB@7k!z2z6rQ_PO7#t!ek#XY@H)t-SFqy-rZLc%V582ear!!1w7y3 z53!V2b5}PTHGmfVHXi)OYMQ&7?^f@dH*33Rg64Lt<+l2|8d?lpa=XvHUT4VI4QTSeijvhIR;nkSbM)T~&{(Q5(KfBAETAnzJ$xrCQ>c|Yxdy0Ic)UH7@$$gtU zpE`wbL5gd_YsxoMJG6cMpK>Qa)zOL3a+f@tYC%jd6*qMkxR5VXheuOtj2>OvU8&#r zURJN!w%4Y$!1Jl!=`AN3VVEp?6cnI3gktZI>!8$KV8jgpYb;YpKhH01klq;c&dY5q z5yf%2!#!cPV`6C?pgA+0wXyS@qiNQm9G%$6|E4mHtlIMv5Fb-*PlHy_q`QKGf`PNs()H9;Q5Lpvabh>KbTPML2Rga_RP*G6<+%2tzHKk?$ z7yRFwD2=VBr>igrhmVgByAKb$i@Oa6mynPU2PZcNH#gfq4mJ;8XHPRAo3jV)e+&74 z%Z-^0^0o_P0k+wYg+#%$nhTv2Nydh z$Nv}1)6V+;2lgMzf5ZN3UjMBQ@E>8q>UKaYM?GmfCo5-{R`RmC7Wit$_i|&APJE3$KBHM{x4;PzyRA=5j89CQ zXPQh!$NfFNzX{@Zaq_5aTB&MkPWr|Xz)!M0(6;C25@yoAA)o!( zHm)Yp>*)uY-CI>bE8{_n;A6l(?Cg&x`<2gq(CeuAV&NTztijiM>}&gjR%`2yPBy3) zJ?Ufcyp?-fQe=K++xm%~#+UEkZHOzaP+(>C&1eXh77(5kIX-aKt=5*sAFlMU2^((y ztSkD8yho%Zr3p$@d5)Im6scHGWn>gZM@IMwjWX3AORD=wC~I=^G5tL9YS=*3)n|v6 z)>hTTGAKWa5++u}JTPckOh>(MLQ=awuxXMm9|Q5PiLaa%4xIEQ6}kq6#j?x{EY;Am zPx|SqB|J@C>ykFIh3N;BV6e~6Un_saa!ypmNW!@Plmn25IO==A|1$% zUDy?MN5SWJ0+#@Uzzr8k?Y$T67Yta(4b9Ii6-L+5xx|bXR{I^^K+oTWiRpiQ%7C|j zgxoF{Z@!Z>O53TsSuB`o!|PTzp?^b7$)9}`F)U2x_;yl|6JE5IrR;b+7Y7|V3vKEUiI2TrSzq|!Lix<1Nq!D5vXpt z{e2W6u>;pXx839xa;%j~tR6kolv*uh3b$E@Qy7I_O&p2tK){tYpAu3@J=V`5_Qenj z2+t4m`MNzq*pIr0CqmGk%xr)y&x)wbaysj$sN~awbV9Ls#fk^(Rpyfk@1R7)RE>}Dhrz~BaIiClxp{bhUc5_fgxBWn0Xumg4J z<_BgHucLdE4i@TWQc+x)L8hOiwmymTUcJO6&PIAb7*xs|G+1Et zD^j|$7R#wI{bF9)uGHT`nLZWJej}m;AZ)av=n&GPJx$H&IInimg0JWp*bEcoUK!zP z9DkE$#u)-b`Frm70h}GCx?JHu)L3k#+9_#Yx#CdY0ca1k|K|b4ONT zCVLGqFMO0>(iSSoq?Uo#PVB8=?L%3J)*Nbhn0F{+SXjfouc9!Ym_E3)OA%bMZKm<7 zx)2*nn9%O|+rO^eB=sf4)ef0;!C>yo~ zL|S3;W|(})s(gvyOm^c4Q%fWD!epFs#q`4gfi}wl+9+KJ3zr4nBBDV31vcfJVsWFW zZXP^w$vMU1UIe7txEVk}YBU4bWvkSUPNLhJ9n)kVD{vf~%2=UG4Hjn{_xVFvO@z6K zES@H-PGdl1SddT~DtX2`RIqWuO@>X15^CU6LzAsddw zYMOC_XI{EUihiL;9D2AyfWZ*;p@++Hv0GT_Xr3t!y@$aBvwyp6@X)M+Dz3xp?MFYY zg=9NF4>{#2dakY@}OdruLXU6Nu8~Ew@;eB zoQB0b?Rn`o*XQfSj^D8u6KeLJ@Fm0H_{C;%myp=hc)QIDdaq*wNU5aLC_|M$jzZ4K zOmVD7sO%~BgUR%LD=RE9$!=~KO-zg3LI*Z5g1klsrO@`T$L^aH$`pY0MS}S1R#8UM zI5)?7@U+B1erb6eFL^!meJntny)AQ-R+@8W!|sAqOb6{+YTNRU>fq;9Ai zK81sj5I17GjbAQF%WFy90a#!iiS!*SanT{XYNdbth3vH6IAf=Wpa%cEl)ABs+{35W zS3>`8n@I76x2Pb&u5vnL8^1`aAFt#rMj9&8R&bNk8~yT9+A;1KV>5$qZ39CgVNuVG zdnrCaH3k5A9!q0?J?cOD&B}w0iNm0W|C=l=&pkQdO=gkFYg9K>rHdf~sXE})}zdb>(*{=rk$k6%(|3~Gqttep$vvYhiqog%<|6kC0kpksF72eBxq1FM}`^j_X5 zbwFf&*2x-xV?0EUf$rJ%(?^8E7+V*{4ZbZSr*EF1LzB9tLwB{7IvhTmBL;!rdMEv0 zD-n=6YH3u$_j`Oc+AbK(nqgI%s^r(GaDB3FE^4jVbJ$Pi@}a4kxQA*!%GAn=aH(4y zkdrB$7a;2Thx@HMHo1x9x0ZvdnZiBv^MHJ7z8zV45)@Isrd=VTbt4bUpyxXDzbtf2 z7t5ER;H1T%v|7BWG|ydQ$j6?B@h{-dtt{6#ay`=yIs+gTI)zo$pc=Gh241`| z(bE=(AqT4&3Y9&vFlgHb6@M?U#7O!#588_Jj`{!anyXd0Dt0!ruvY1-!PiJ zMM`ST4in^DuIm9=cNLWV_%rOZf7x@ExWpf5L6$@6`(K!}V-?K8dO0Er`oeGBTn#vP zF&|*7aIARybrB?yEY!Jbn_y{mm z#tCd#2=0$&oLLOxer)ut0eSy>W^}(SxJKF2IBiZr6z8%>Wm_dTgano`E|NhbXeC5R3C>>ia zQ6;jnDW(;Jq zn!o%ycGGqDVix7tIq<8x1YX76hh~k%Y|kKr%3F}o&9UI_r*%p-xTf{#AYAp0LS^L> zC)#(C2s4fQ9DAldvm9wf?UlD{%v6C3AV*56s$XV&oBvi%>bko{VSsCnsqh|>|s)l~P%aE@ex@#40)n5&CF z(yU|`q=c_$ox!N=M^lKIZ-)Iaq+sLvKI(u-==AYW?>=JqPiw+k><;3Kq1g`U55#c0 zYsPzVeuJjw*yIHE@TzbHD7TVU!N@ABl^uI*?K+aP_$q{~&M-?qNV&vFtf$pg4{h{P zpuO8u6#&X0E(Ia5D>1C3mQg`ez`RAmB9Vq46sJ<)^3u9uVGfsLOmj=%miM(|%<1F# z!9!Won$?>ArldQInZf!xOp*WQiUTNeYHefD8`LDy+2x3;?(OZp1`a6R*x@47);mH% zn=BTpE{<8!6u7*6qEd=aqdq&}FIN)`21sTE+WP!nv(2`T3w|f8DGm(6_=Tp_In6*b zKvA+y+uEh*BH0wGfPlQ%SJCo3Smob`={DRLMbzOdaUoBjctBJP#~egB%^h-a9}0A} zay)wG_q#%5U2>DlW)hPks{4{)wrwn@CtXx)d^OKR-rEr{%jN?<=;gUr>DX396~u^j zVp~1fa9ZpAkvE}pe)UoBWSDQ^PP7G9@&Ee)pOVOOm43c^RyCnATq6l5f#b za#_XlB3X0P(3wQ~wK=2#-{%6PQlZRJeC=$^F0&TfbNniih z)2eSu{rLh5sSeVEa9Sqc*t;UQWflh8sYmRK1a2gxeomRZ(dg}W2n*a4nWP*%^JQv) z9gQY`#j2j}OD)r`?5#_o93x;C;mI~Bx?PVYM$f^KJ98@8VaGoKrO6e5i2w zrtXZ49M`a8?yvkY31YDd%aAy9qK}{yb{Pg^lq%aUe`8O$&Qvcy5cl4;>u`bbf=}RY zLnN9ZDeAopA{s=Iq$aJAP;88HPOOQen*i}vFwBP(M$PW)FBH7I>+Q^?q~vZ@U9Cdg z^%+0<7&WvceU|ckY%S@u!a8l3D$eL+bFD!h%b+BUR#eM&SW^g>ZviNKW`XTW9u@2( z;zP7LVJD*B;UcoP)g|IodHWdq{?*A})~CJQnPaQNIyo5o{Ia?7@p6@(QkjA{aDSTV$6jo{SM zhtRM%2T$B4k=6ABeT!aRol&_zaqOwlW}j0u zzC+3lA6HZQJH5jr=hyyR^t93E-!UL~M+T5&&C#D8pz8}0c6(-3$G84Pm~&{>=o#%v z8LiZ(fAgz`2(6Wvz6|`$P?vmzQ^=C7{7csWo9e{RHx1=S?vBl7=2xa?RMfj!2`I|Z zSVO!U*u#cR3^MPs{@$rXpw|Ic9Ua~*4)M>;*QLh(wq5o*!&%tFv;T>YB>wn26xcyo z(02RWAQl#O-TcP&HT@!q{oz^Ec-U?8H6!-<=;vVr|5k3x3baE&ON@6Yp}}sfu)zvj?CpXod0rzF+XBUkQ4D4$nr02&oJPcfX%^kq^z_rcKjyxSZIFQvG zmzKwJ7rm7>7fLptYD<@8Iflc7a5$ zmNYjnZ2I8Z#Rz_-9W7^dX(GW8)MMy$V*A4k@1~BTI}y4s?rgNRm~*(hQ_=z8lE_q8 zCq7%~Z}AJltZ|#iQUaOinL5bA<~va8pS+?e`f$aKKxm?8K{&iR^VtL`1nA zH-E1U_Ul*y;D|{3{GW5!_L0u!Xt|A*4I5UUxJSdrcOwD{Cdm7WFaJ?K897vb3*lB! zzryZoGvdYdLzSq%`|_qqPRpaqH%B>OzX#xI(4pNi)--|m0j0Pzlc&-et+l-uit8;- zOh^MZ0-%nUTTQAM=M$-;^v>5Vf!FRFZdl*R)G&`j7b_eATGVwYv1^IevBA9wc!%S( z`tzC93YVD>bllm)CoB=c+C!2MjzGr?69Mq8`vm5*oH;6rNHZz&BQ|0bBKY|>ovivS zK6p$WAN#fqjZ2S!GfPyJ?Tf<{D?_(jZY8(cU?FfuLe+L0^+K94@DKSMzxLJjz7l)i;^L6;!^y?Y7Irtl@=??j~0|v!5 zn_J2`);8cH`nTlN@|!@}poeqNLt1|A-(GiH#5#eR-cP9i`=5 zL&VYcQwdXYIZ#BD+EblNDp3@0iBb6RhTraz-41w0UybKB(_E7v$OJCLGfcRTk!(eUShSuwknI7mOIxm zc@Jsf7++1`Fq!2uu1)tDD}ljOw78_Cn0|G#i1nOt*)%uIWHowX;5is$X?GjzJ*6NQ zDv(*jcZd|JBBohO04}LKZPsE1jbys|+mz?#0Q!lt_HvBs zIVZm7Is*y&O}3A%Sy#rhYFX7+S3Zr&#LC#HW!lL_VyH}y{WPOecIhwNYcE@35S|xm zvOA*I*g0C81ckJxe+`1QHe@gy*M&uF{lISwHuWcov=ARGwjSa{LyI;~MWOBq!J90i z%KK%pp!=h;17VxJPOcw&@tRRo&KO38`sY|~M|o`DHy8zrtVMYpcE zVqp>m~%R-rh0KQ7#2+M)qog?lsn|l3k^IY?oRKK${zDRxtJEFn!L* zf@`sJXUr`RLQy^KyKD`6m8k7%u}A4{CEz2Hoe4g$s@~1U##*l_dsff-;53IWF5vB~ z;mY^*fc#`Ig)3scZ#h1zV7X*8$k%OR`a3m@zyJZyZf zVDhyfcm|MfI6i$U=-tDKC zxuaZ$e@t)JXT(VM=Uo)PcFTl|{07?Hv-^jK6#EV%W^`uiAua0^U%H$fnqVVzkN@a6&JoAHt4UF?- zC^SdgHS||{FG&&xlzm3>D}>M})>9lFTKk4U&VZ?NJQ4n?AQVzSE6IZC(q;2aVrb@D z{H0eEvssTQhoe8>2{uXE8L67@A;pCH#mEhPI^jF>lG1{nj84Q-jx$(fqfUn3Mw8pZ zu(d<9yzrMGPwaD9o+XQa*M&E0Caa+}G6DcBP9fYX{JB4iJFn!s)B!9ipppk-@U*ce zQ*XFiR4?f7?+*lvw0p+lWE~K_?BJp$hhN>IF4FZriuWTN7Z(Sf)H;GdAR-1H-18QC9Kx#U^moVz zOwTs=&z1$_bUn`r#BO-6eQb(>{bty=jGKgv%G@but^5=Zaz=*8jh#so(wigzb1{_N zRs!%A#?H-aAgeW6m&s*UiDy3=^yOS4mk^z^t`6(*=e^88n-c=YJ>{J6hf0=3`s?wS zhL-c)L7(xP9iOXMh0#OhWMT(v8qK)R#8SK^@rp4iv)a~u(+z%Oyok0tgo?*4v}zee zs+B67V--`jxHn6~6Kp96@_p$|A;h?==`?hiTgY?ZF9>a-&V;s{ZN|gj(K|{+CKBlZ zB7YVrv-f;ArPhO~ppP{fGTZK9kN|NOkGOP7ny&Dqf_u{o7wnU3GvpTC*)8LEnR@lN zwG)MKlWD@y&gGX}Bw$Am>vBS(X|^@nWrfiN13_QGz@aWq{QJo+n*?C?HDu-*cb0;wK}zOgIN^uOJjFSkJXuNzJ)AZmo%~x@ICd}>r?GAfg+*9nG-)?2s zOJ>C#n80i_{eONK z;v#Q9gUy>7L(sWtWZB6}>e09V2|2sXdi)?BjDgx`b2zZ&<{BIwiQ1b7$>!PgbMc8~ zTrKM5>ku;bgw9oBUyDVy$w^)@S8m()qhoo<>ssk&VdpyVIV0xC;u9q8Dnm8mwq9>W z9l~|~W%=Uh$l}#jcj*)L9a>l{$8Fn8>=EB9UZrLwZlyR!CTX@;uhjUk}(EPyZ?nzF4`R8eBEdqq9GCh~Kj{L*2HTV&V&!c}|ERm|;Cu;$L(8 z{Ui;;OjY4u`wO)Y+&h8KOET&ksl*eM(CbrMaDQ?-G)q5s^rIv)*NiD!$I}8>A8G{l zB05-Wxm=xCAO=SqTD-gTew>A{zy8!a-@(Skw<;|3@!^EiH*d>nF>1bwkI&;~*%(4{ zUW3m&)T{}sJDdIEJcqyfeo}WK=;%5Db4$L@0T75Px?t*OD{V`mi`{xR4n1{M&ACg? z-gBvs3iOA?Jo4sZ(rK;c6wh$`irT?wbPe{N7FvspVpM)WjR6t6>qhkw*!d8r9kO`JrSL>`>Z1-N>8;hDY z<6~$HUg8hs8k{R>med`p1Ww62<5zHNrnl&ag|HJCw~p|v(&DPk zJtvNQb23OH#B(mS0VWssm!Chg8z0@NolnFN5+R+2Q27fcF`eH zYNPU}FY5%jnSqmkou;*gm706k1;n<+`zbh09*!1M)na_hrk>c9czk?v;0L}7e3J8N ziFG^T+Hmjh5vnMtS+)4=Hv+u+nbnG$>ad8PA(BI!^K8Gt(r!wMGo=#foQZW_=L8&& z&L(A$VTqj%t!rUzy)@?eD(#Iq>D(%-P>Zq8G@mK4|CLEfJ21^g_#6x7gmS&TZ4g*M zMkcJrJs*u=n;6+DDo{JkJU#_~^--(SeSB6adw`2FyZj5(-VW^Mi)E$ru z4dU@KY;%@*W(vpi^V;^3=`?^u(HU>ZKQ$|y@QK5ljBhp2o++=QtrS&;Y``_dBcMk& zITwE@&VW^S>&tQ+M817XRCVcWlGD|4a~tf|{C)=Ha7bBF9wP*9;Uki4)>}Xzg*T5( zD-@A7qG3D2kPtt2Fb&UHLLOPm5^&Q2W8)$(V8+W(SX;&jRLVYkVrX04DFHEz%EtQ) zxtAa^{64E-e-z~_9sGuzR)tMju}gwUkd+Wu=N!u|`TSC6a66izgIK=*TLdO!v=Ils zVLDH-MI)R+2QB%bRFxzdM|1pD##a1*#noOTE95s72lFqn;#kG8e9VoRvT{q#ST^hK z$jZla0to2ip}RSBAKr)wwd-t$f|F}|9c_bXV+q|t4DImAw43}|$W#JIn6RHb1Cy{{ z-@>SW&`wPvHswcEkn#hNSfcsq#Xcy!H)>1}Bgpywy#FwQCxXkkgSK3+A9v_G;(B4F zoGRxa)&0-oB#lSCBwv4<4*Rccm=| zW1bsDn6v!rslD7D+6{PNe51<|^Am4ZwvNph4|#mZl&)n8e17k4jHCQZY+ZKk%dVb!2k1B`GjMW6Sz1`r3S+hk@j_xKnEuULUhdc+UHMhSVmTYX>`}uvlp< zt0h4QL)K*T<0(X{hSjeU?Xg71b0e0_!{h9VXu%-f-^n~G(!~7E$Ydi#5?54igBeIA z9ZR=RDR^L6RY5)f?KsD;)GU%Z>LW#6+;`aznE`3A8<+sx2Lv1f%sTMGhYKFJ?JT;< z4&!c8DDdl8(U7peR5kAz{hUTGfn$c8aVu!)7DaR~-2-u;&_aJ`BB-UBYZaMgQpEh6 z+n?BMU>nlwh6ruoLpty&Y)4oZ&l^0_h4e<0$i4b*ZK|KCrsY!1FxTLfEtu5 zC1aLUz^R#Ik&b18a>LY#{d7D9n?D*Tp4k@^kSjj&$)d|l)1!qu&FYJ`sKANMThYy# zr6S+uv`G;gM4!yP*8RiDxXmxL9fu7(LsdXAz0eL}$@qn(^Pi>J9v}}(*RnP_^5vG^ zO7k;T>b9@I5upJUf<>F5SftzxLAntWX%&&Q$$6WFFr;=WClKmQWu*w%e1R{P=FAf} zWKV`NX!q9&h48BUN1H?8J(pSRzfD3?@r;Tj?noFcOG9GaWOdC^+dG&sA%Td?su*w-| z{c)Wf*A=YMWx*c+esUW5nQMdc?e_8S@Avx}F_%~1AJTX&qumVvTSE^NNA7ZoGK!64 zaLxI?-S_zl|8g{eyAX_?UY!i~rdr52Yv!R6C&3bmkDjIXH3<~z8i-4mQMj5kxrgc+ z@k-S6c$oYr2oTUT@^LM9PTBpOw_n)rJ)#jQ7`8ua2U}kk56l@Eom#+~@lS zPS#E-*4H4EQ@3Cv;!t$_HnYGBf?STwtIeJFRgjA>;MNH*K!Mh%4l8iNB72S1EyqKA zhj1#tx#rvRnD^c;uf;Foh~}w~+SksH_2Ab|?-7=c!c$*AF)z^0LbHkpE!6$Z`7I0D zNMt!)_T1~VCh0?3ORtHrVXZ;G!}BZCWXXnycg6txu}TCAJL4}l)D^cZ1cOym=$A?p z!Dg0}Ci)nZZ>BTH2DfR$7em|P(e|7ym8>!xR3L-06 zOeX?VCWf7Dbc>@5#kcX6H-yRd+}Hy?N?I*>d}Zzs7FWVjlYTn4@Q@!hhIUbOOQlRZFW$#ZX za$rA!^G}EOZ-qJ8D|WB{R#to9!;BnRLq0@9m!=oB3fSMbK-{2D!sXvbM9t8z83l&- ziQ^ESRkFR5=*VG3rSd8#yM+W zlx$Ocn?8kP`UUo=jtZa9CCaFbDLOBhM!%h9SiWKIQX_lPNPX^J))DQ37|-2mT}YzV8jiYZ zes-S+3tJQPm;mS)*a;);AliVW*hD(zr?B_{Oh0n0;yA>>P+6J4c=b?M(dlRe0~!pN zwpsG2b*oL$`fe3-C;p$JTeD=1CWT>xjHSe^H4&D#(D~Dj2A9Byjtliy1Gu0qs%*h6sVICd> zwZx4;%TkGPwV-ruWzK6ul#px#f_38LX(c!k zxa!eRp1Cl9czNnEWS(B7E#o|-Aag=Zmp(_Q?URcA#>u0L5q0z zz`L_guRRxo=5XFM^9~V(vqJ8Js54HhgZeVrn4}{_>=3ctGPyhlpV12HsUzcDSIbYO z5Fx!RGoz2F0$d_YVXRWnAZS!tb#r`vq#upKu~u1a;i+E?q--cAvq_HE-iIsZ`|tEo z>%f%W?H%4UyVp8h;9;mi5_~mx!pB&am7$9pZEQ2&A2Gyr%Sf5#VI)&SM048`((e*X zZkU#F&3>Fcqol51ehLYY|F|n=l!x_*?$wYsH4V{0gzFQ^5kC+2e3PKXgkaZRjTr}r zeKSfSR}I4dkm`z9+D9%TIj^;CacCv2*pS6wbse=rv;YVPS!)A-#Fr9wY(Rquzdoc# zkIgX>?d$}p_6zn-SMcKDmul>{S%Q>j{WRQm2Wc4Gi*WbHTZ#3ze&WZ#eA_m&nNwlC zWDSTxoy-}LUqo(0CQH!w=;7+|vkre;sC9tE9qaFN2~3se2aKx+?~_De$N|JoRtBBG z@ry~{jO8M$l$!^96qwp$x=i2xQY%v^MT7<5=C8u$oe17v+!MMK@t85){7B%fJ*t{p ztI}LV+(?P3|Dk>4WfpWpDPKyD!@!hJq0!3MTX}=xAQ28R%4BZ}9j1RMAY)+&P8=Ai zsf2?YLyC*ub^7ShHa@N0S56In>4LDfw@XpzZdrBTza-myd(3w?k3ftA#(^7Ba1s8J zz<}wMSgBrzcsbdK^Z>2)dVUO)b>k(pt-Byzh<28eb3>|JzlIiwi;+UHI&JAaZ2T&} z&WWgXYthJ}E`D0KO8^8uA(|Ix|NFaKw z{4rCN{1*P*u@t>pCdFZ|L9y@YisN-x?<${YH?p)xia)QJxS4>#Od3=3{TrD%-cNAT zu(}>6J%cg4&U2riLg#lJN2rNYcmrTo&~lhkCZ2O@euX$X6;V)lwYrl%NN(($f)k9X z$($}pq{Jn*OD}C8XTqbf2X&g$56#+gc1~`I)xB5H!$`Evh0r7T zq9%4U^tm3_Q85{gI!&ItpigB7_6U7wPnq~no4elSFe`?eG(`UJL#*HMQNNL%F{o>%1~Xth!hwY#aM;DxJW(ow8u(gC)%9n5-H>8b&))dk&>ZAu{z6{$JKF#V7Pnj zVwhKCP$oNU=l^T1_ONm+*HTfb%L@s^WLXo4qRvtFCy9i=#95-+Z=MY~`R^>#=e)I3 zq*<-^9VtusLO7h8uhs10h$Tcy{GXX56{)=POrbxeIt)#aV&K)GQg$Ri1rx?;+<8S~ zb&?&IVFBiDQcAC-Q%i~;Ja5$b(?A2E>sQ13*r@#mXM`@fAGk*8W3)t6JXZK^Q#W90 zMms>%@A~AS`=8b0gL`b4IXA7PsEd}~`^S|2d`v@;B>};{&XP@)IS7^tXftGL?k>{C zURp-q_*M|g{pRW8lP=P4w1&8YgIMzF@W*PP3Vk_%(BOqyY+*XSe{E z$t>~*g!6!Gr7IhA`8`tI9J(=e0UZM+afWy2`0ZZMOTLjHD}!i>-oHLFHrrZcX!`1n z6a6$fsy2`!qr?YQIeimAlHZ`cbYl4E(L$r@oN|GaL$C8KPBRk%Si;llGU6aey5O?w z89nJ{Y!F2S)|4tl9(tg28WB@nlaD{kmwU-?NUDiYr3l2n-Txpd<$kF+jE{5;-r7BH zCi)8S^&9_n<)I4iHhVC_$O@*E49(N6X1*+?!B;SvolVuivXW2Vz4;W2l#kJjHRml1 z@0O;gOe%JNdZ!@i^7>sjuiIy?NL(kCayJ9&tvQ#BspRP^Gy~ObP3+(u63xpNJc%=* z7=Lo#vrQB=p}Q#oeHJIv9!C5IaRWEhk_Fx@Eh-B9SQ8fFMtD zvzO)K+VEmA7X4fMl8~{DPX-%c$m_~YL0WAO*K=_JVTs9V-#|-1SNdVyRF{S4+XsKP zaanAHpm}-MB5F;g#ItM|o7JKzZfZ<4itNQhPBSY1OVRvQTK)!SB>AwL{GNUhHxn(> zCt6HHK8$f12sFh8Bdkvr_10z`yWsx=Qb4W0XC3s@U$%Q*qO{=5Ayuq0WH>A@dW4q@ zEx)J5G+2*OmugILfkXl7U3tApq;?3WHzc#`zgmVQY5)9}pQVjFnT=vC-#GqXB}bDH zBm|V?m5c-tE1i25LNoeDMq&+E5+;R{=;@I!T8t(c;R-R=fk&jGeiN4uAN5#iF9SzK zBD;#t9bjGjS-3NuQ|?hTo}IOXRQw!Hes1%d(@s+ zba;}+A{gI!!^V@LA24D}TY5WWi;T2<)*a!sk%h!=UGBS=)m=UQWmxA zi`n&LRSrk}8J~J{kQtBYvdqo-Rc=Th%I9<>Hx}!k>@9C`iu{&teT7UJK&UhIqUETw zd@%5Ir6qtUi)i`7#ZrZQh494-y@)f6OiOXNsvAHVM{E3gncT*^Cq8DB-AmrJ$!?6M@ zL2XzO6D+L@<0Z+S4%3pRaLq3uc*M$eRbG-q$Ok^+*AkI2KOC(cW*W54|Ue$JYMc2e5pEJ5p52oH=8169Wm3$s#6XrUZnFh#sO9OIAM~~-Ta1I@C zjf`Z*TTx8P)w?my)NA>{+Rz)utijSK^00^TQ=OKLhs=0hUo%__;7AUxlO*4*?E_Js zg2Cs8ryMl)CF`FK8GLYR7jL)Cp7K4%(az3J+tX*q98KeCygiO|QO9T*qfR%kvGIVv zO>fXRqNU!}YO;(%`N2uu@-;{vV8~>kTI5k9cu(uYdqirhh0uN%4Bn05it3koA!CfFaz72FJu2FOtQlU{RC} zJ<$*;odiZ*BnXcvlSW-|9RBw5B+nXe-1`taYk%ydq#=}1>%)W2FrGQMb9Pm(SmIbZ zN#WyDZa$Cf^CPkjon5RZMI-ojRXt;EH(K_%v+s09u$9p45s}-odxA@dUG)D z^oc$9i}O&^29prwL@o_3h1@#OKl!bEfs8Mo% zvQMe(lBl$#W+h$mkVUtoH9T5pp$#z~L**D0N3iAs=z^0R$t( z+8v{;V#D1NLRre3?-VGz!VsQ`O!5T-w@?xDKj_Lr6u5YSYJA|zPnHHeKrJU!a(suU ziSQ016G?iNUnL?{k6SJc!uKs~7re#l?2H9!o(ahSUT-Cx-UdVPy)iR(SG*g~d4c8o z(=%Rjbis$;9pNB@Gi8tR+mp_rO}SRoMs^|rd={c-E*&wEf*sX>xfgaqoY^}=W{d!9PW6!Qgk|o-_ZRq0 zo(NoRuo93Ttibzv;~=*jGE5;-W+O-CAK%6_66+vt5CxJ(`m}FSLijJ46!0fp$Xai+ zlqZRN&~GU7WkJHZB&{b5JB5}l*Is6>#(OX=!xRK=stA)R47l`i_RfdPz!hSadNn)r z|4}Bf(>s!B}yFbCN6fu=Pf&!B9$!aSeS2 z-W^YLr7kh9VGyYlNHM^$sQO9gaHCp4Oi@_!;0W=ork9KkMOV!xD?6!;G$dP1Nw zE<$5++TT8LqR6cXTyXG)bx z$t7bip4W6Ab`xHXC4CCdHO8kL9C*qr+}y43(CcPaqft0~JihP#vt8@$s; z@!D(no@#(B!Q}Vu5Oa^&>;dCe57RQwV8L2X$8HDV8ArF~+%zjD64+*)%)wx<=!&nn zX=WIuPYB-7b$df?v2}OLlGMr@uNn_7>OhUFl+sP9S^Wd*K_}wyLcH-;V_#AdR3qdX zy@$@cGB&&5;=9IiG@eDrIZbWzvVjqiSCxdoBOU(9NtTQw zbg)bKN=s~>7&SyMg2Evu$+D&g70PDtQ-b0TjJr5_^(AN7g9~r7Xth)j*d4gel2HSK*xa&to`wt7q2hcpT2+1W1zeVm1F-`9RGKq@7R#HWa0_gkEVeSIa=?V z5Ig{T!mHbSAB_vwj@vx>;%huS%5MYS(O>HA-oxKlz+EH9jW0e&z5^x}PnpU3I%0Q0 ze6Z@CH#*Fbe}(a1@e2TNH%?iEX1|+lXII3ZC$pb(L3@{1&n@uy zjtjfJDU@1?PNJX5IETJP5l796sq!kFcmV@auN4ZnzB8u{jXrtsDdQ^MMW>wnhj(YQ zR<02GC-0cb%!RKMl7z8g7{ohl2wXEjfPsr=M6an(hGf(QBdq)(Zeawa(;F`e6;+8! zk0Ls}lLiW2;UHgbc;!?E#%Dy?2chf2V0awEF^}F*86B&g`!dS+jL~lJ;yfyObcn(2 z_~w0KOiVCufMQ(3%|+ zIu1R@h~Y*{GVI9P7i#Rfu4TK%YUAM&|-G(<&{yR)s zKEuTtm5Hzs3ONp<^2q3svC>-1VmorEJE#odK@^#$c_#zuf`){6zhzS4RHDi;T?R$@1DQGVWfxYxr5! z1*fiUz)y}of|ji?Jk1s74ey}40`Ckq*E|3li842u zE;yBuB2sXr%QK1sHXcABs?K66G#7%#VTkYpI0XG5ol3@JN`|5<1&EsUD$mwqiFS8R z5sH1KPkEEUQ+^2X(3eQsThWC&c5;8%htr+??qLXbJY4S2gT3=7#4tc7C)XInZQfA> zuYJ}4nS`)T!X#mz_sDP($0K1pD?AKwk7ru0SV}p+dC%F!cl<;c27Sl+9|sF&Y^902 zhrwx3F3DuAoVs1&L48H>&Ys53+ZpJ(m)szZTm4InboO$k;f|enRc@ikucHUQ=XI{dqw|wfx)^7y;N``VZ0z4= zJce7dtMs4(Sli4beUT9z8I8@m|1iJ{UJ0_t0RVS7wzfEz$Im$MOA{F@`kFV#n@#{+ z#P&(I&8K=P_~6xOVjL{K63cRZ(8n`xkn4=~Mjk%St3D3#UOP-AcIo=JG5meD6Wt$n zM(%oFe!2{Lt6SYLi`DS-wQ&0fzEhEQ!V7wA>w@v7PwDYQ7rV(kBAQyMXB!O$a7j1J zciKcyU=R8XuIRR5Wk=WAE5HXjGZo6V7{TT;ziMYboyQyZ-(eUzP)eg_ z>yKA9p*Bk@0Rg=qK|ZPYPMbT?n8f%WQWt;d4~*Z zh`w*oO^hce2krCEn56LR%f}ym%r64GUY=nFiZ zH`>BSJ$d2N$#0&w(|uL~kl$O-9w%)O4N>7o0Y#{;(GP;2WN?TL&skOj)ZD^w+pFIg3h|oJr9?ARx=7*{jP3&MIUyKx^rU+^|8pH$m@a>7fDZrX3Z`D6Q$Uvj$Oy@uDkIP;57PTKQh9urW0 z4Tx^aw`3_V9WmXx6BfI5-&II=%2`4A@3Cxhc;LW}v8L;VbpIF3yk9X3fBDT1?ekBc zwl7}t>fm!J*{ep*JUEpzj2Z$m_g@aEW?0y~z#C$Ns;c&PS6xsqo*O9aNn3%)cxZ=}0aLue4n zvvqzXBZ_n-QVEdzYw%ETKV5t=GtQjy%V6V4WoHn{v0CKh!j&(6Jn^pm_q@jKuUUKh ze2Z81@>YgB-n`FLf9!>>D;deRk5G@avri94?*n46u_NhZindYdCwH+TLdpbHwNKk3VHO#-AJG2pPLK zn{?q9?xtYa8$9!T^4Gs<^OH|_@zL+{mRMf}Jm(QkcU5xa)yD5%o$Wr1nQ>;Fq-M(wAw3>;gwgw3D&PM2hFz6U+Q0kTzim(9 z`H%nW|JE)~zHWz~{&zV*8=X!nEA6S11l1Llb%X);>P@UNd;a9D*4iSv6_?1*a4;ts zqr3?x5gF+Vb$Co#;|<9GTuU?|mb7dXy@Jf2#CCJ0$)1EvP0Tc(|9qr3J?JKT?|jIU zM<e6+de`3ZaqM;X^1i*><(J!8{N2ZMJ#onf zWAMe8{=|gqd(WOe<=Fg_ye8zEAHHwD;KvW2|N1a%f^7D-Eev>$S4}s}Ank--dOtp4 zR|9Y9>wY~fz?=btVUj}rF*_^QEXlln!!I*wyo~LfiO(qSnBOhg=fK1cbUU|9gjgdy zIr%8>A6#ugQq(6U2xuDxT4^}YZJtO2A zWhN#zoDed(hFxuNxMG&U<7yw-!p=1IHd%`1Kmg7F*+*3XWf(7+lDGg)3ve!_$0 zyIei2;E}J7c{#d2VzZ6++oe7ptFpUxvg7RZGv3_riO(`~bjo$d8ABL{|8|+SD1H^g z+Rl$>V^2&p&e$@<3+An@09XWP6v?kN@jgUg$4?I0!4YeP7~baWm@*s#`hk~^6+~G+ zkiV7j<36wbxaCDp_sDm^QqQh$^1?&rfCDGU?Ft=I>1jr;B8!qy!_wY54f8Z~TuoWRKqFFg%f-Dk)3@-spH2{vomg_JFW zkZK#xe26Fi9$C+sg*#t=)Mf}3b%W~-8Yz`keRfZ&+^K+9P)3_rr+~^^R3f$P867fR zQbtcn^H5wDqVo>CymX`@O>d$HDK99tMZ|E3RE2UY1ZojOIP>bRnyNSD_430@8bc~) z2JFJ0sL3I0>Z)N%)e8wt;$^UrM-q@HexZ>Ol)>emPZ!CX_A$tHiuB1N#p%|84hm%7gt=7hqPVj^~wws9a+WKa(memyvt*H>KcIA z%<+WX7{{WjuW7sBz`@dAQetC2zez%4`h4+jUs};~WZ{>Y@)q>&|1S8&r(0gh>#N~D zW*KG9SdF=D8r9_!8`=k@mY&YRVl-|II&Vp>Qt-)u$s0K#<|{lx(_`|b zZk7Ll&zP_T?Q(Te7%0FHdQf$tRD2L9z@jW7Ux}>K^$exs^T1C)NAAi=Q608^Qy6zV zG|I~zJ~Sc=GV|MbjJZ@@Nka?+VdaeXTrvjaSC?*Bo;l}y-=#m6&6`p^w5k!@GlRKz zE5%bUoT_$==UU$uU8QftsJdHsdT_861My{+E;z?X zS=-xXZ}^k>NqhI2EkI;nVT@UOrHfY_J+UXVeF?i|+0(cL>bniEF~E1of5WW%8ArDc z*dg&JZg)R91jg=6h8W11U}LLc6~tBjb)5xb=A%HGDx_6F_T~|NOd=kklF*P!*HVbb z6&8bSl^sy$8KUrv(SjUG#ekMEVIe0QPYOHJ{VxpCr6xQuU$LIBwd#|vViRId!Eo`phY*zlFRuY&)Tb38~h^ICA%g^ z^w>1EuMVbhH}G0ZPs;#vax$aqbZ5k6947`EtBcw`sP2-^KF6_KYB_uRie(&*h9Tb! zFKUI{H})Nao;SSXP3G4e7faXdx)-j>Ny!mi{0%q5JqIMDO(VkBUkkOo73+od&c$j6 zJ)8$4PI<`vf(g`99#`=FI=(L0UmI94nYqJS^BBFz3F-=C(c>yuRL9o{f(>h+(w!MW zcFG?f8csgq8>Ti(f-mo&-kyTOTN>L?FW<>wTUfmeJuG=S)EXKKHbNaVw4iw@Ej@9MC}Ra2UN3xZ#Yy>t1v~JI+&ZZI7uWeRnj6EtxNF}l@H|- zEWC?$=mpm{q2=k>hJ#$?0a54qqlgN^9J<{ISb1+V61=6p%_eeLrobjw%# zB9un$D_cFKf4k31*x5f;vMU}|$D;Ab9-wgI!ORvzU1TT$2G_h!SRLUJz3HsCV@W42 z9*DSP40y?VW)=)9_#H857AzT^@GAxEv|R8q_9fl&++%Fam{O|RMn{xaI~0K9lDH1t zr)>s$8BSU~`Fy4EifE94-QmS1ECwHu*4rt#Yj~!6c;H7bqQKd2Rbc-j^+4@SJ5dar zUJr0>>~)&N!$%%vB#BKtC~?FQKI6+?-012@Jt9i(UX4m_y?kQDbKHU4>76KEFeCKT zCf;eEAFn>($kTg0nrHVgAm3=J(L1iw`}t|1?Ok$k)!8t%%V)-G+D_Z={^bwt*I$3xzW(|b8RL0pYrG7#Ql3@v znP-1FR3~C>l-8Rsvvmy1bz8f2as82<*Kbc_gqQpr+Wi4<;$p%4i%&mk-{r6c@_XXi z0=gbbUk|)9j?>|g?_cRJ--+kOCssNzUk+$8*pVfNH0WMqDrp3u;(c|HGuXVn?i zt&eCa&@y|u!y~N|C94ZosC=7FhNX;T8E60l2vX5_Q-T{B@q47xP8ppzMjMm|yor%s zqcbXRs|%0s|L^~uZk30gm?g(w&}c5Hcq4cK;g)fivr^CX z`9{=RoHVCGb0c_zAms`xJ>|6b z5Bxgm=>=~N<9y=hpZ$XOZZj+9X~DNFUccD)p(pmJM=STM4kEhj7xkIZr{`vX~LQZ9fJgON%Ey18VEY~|&!|H+BJkSE8;1uH*_>OHnD+vG{orCu2)H^ zC-T%SMIGptJ>}!9_!Zvx)!TC(%6i+raGkE{`%v}=t06?pfo6IPF=F8x%f zShN#xRD_%z@__>EgZEI~>n<{VxWdf&4JG2#l=y zY&@Yi1XpFqvm_X)hUzwsviO5Q|N3wLzWthq{AT?-}+iL!axo6P`MRPnjw*B%?Gu@TY4U#FY7Kg?EHe&V%Re8MzcsCMKpV2nI6j zOs5Kv6L2uh_n)tzb2Yypg=n5&iR|Y1tGh~g<(WMNVX-vHN`|0yB;zAI#KjxX9uFUY zHl`Q3!B-LO!J zPQJKAv&|Vl*BSjeiWO^&?!45Y2!D>&!v^fKxXj~-8`b)NJN#Nak3asheg4rG?U0*fgK_ALMeEDuPW$~SUVD@6NMtNm zKvB@Dgu0|MYIX|36FQE0WpI*`tI&oZdc#-p=f6khxPE{|86O7gYkB5KRYur_N$V+w z=>Y>tl2mGfK~B6_D0WZBm^1>@WE;??ce-|Nfg$5Mi4a^(#!UZ>ftezW3-sFQr5sEG z1>TLlT&AUjist0`4Sq)AJNC7{z=IreChroDmfsR)&)DE{PbAFR954+M1D#-4qEUIIW6Oo&Mte%`#iAa4@2H;9Abd??e~}L;!rQ9IpCma zidA1xD`~ma(hH2>!M6xiUc~|{M`4~66~wKjw4xgD2aEynT4k<7QjP!!9@&K0kxjnN zj=4HkiJ=ACnVGOqnw_)-S6qWhEP9&|I+Xqd%o4NkiavNMncVss5g^b6ZkB~Zo;$@B zC}HB&22NlTMJd+qL->RP2Vqz2_5N3Wc=1=f`Tp7dbGqhJp7v#35X2c?Qe}we8@4>o zW!j$=^F7y|lwa^1@FtN=JgJ&S=sRp213D}A)f@cqQ`P`|26@wisW87&cD*dT)2g(!W~qj|{T7<|AXH!b>(H9cKNduOKaud=JRTtA_7T>Z$Qp5yW#7bxh#m z>eWS%-xMN(U$z68iJBiyUHsx&dHH@|FvX+|eK2tRi`<>O*~_r&3Dyg#ufow zUZMM>Iep{7AxkelY<|V#0C}_$M-`vK8lbQ3@NKY5j16~%L0}#9yBeU!``zbVYDM7r z+jeOLr1VD+XBe;D`_c!Z@QC8;44uHlg((ZTOU91gmX__|+$GzN7mHc@(<>h_V6Bqn zqCYYle#P0xC0mf{J*@_P9c{y?RcaQn6#;pA8>p&SxmQBcLy5U|IQYpUxdyIG0VC7= zj^Hj+?4bjS_d4=J8?TYOSPv2c_xhK>(shH1@Fu;(Qdr@Sk_X7lkijTt6T6i1wf+WJ z_5d*d(uf~Q@=#^s(ug3OFrtCgO(ILPLl4~aR59V+ejwO9*GxwK^Lq{f{PA7;CEezi zU-NA5F1r)=Ec#k62tDWLxX*{&JzDRZ8hl5ghU_UDA6#-L<(f6S70V<`4yUri%+d_U zx-j55GWZ+{i#zSslRB!)r|aPBuS01NPKf5*6n4)pmg|l+=uAp{&2gU``OEHDGDgfK z2pMiL=I=Q0`p@hcU-9;>Ydq?brJ02?BfE!W6Hs(k5;S4UgEqROg3*m%d^F0T=OPSP zR07|_06S5-dKs4F3Y8WWgn|010( zjAV*Vy3kUw7z&vw-@>FLXcT-*%d&H=ynKsT#w1e87)T+Y+!!(4nx1AYObYE~j4U_e zEy#0)XB?0JFL;!f*YA0|eQRH0EbJtw>vb{`Ts+i_aUUx#?V8;WUjyRngk9_=tDU}l z)84;6W!J)|=$L3=bTf?5U6RXpcuh`8@A7kHK6}D`OuJ!yj~u~(b*z|Hm@55WI${3#yRQ$a2Vp(_=F6=soNz4K8Q zLv49lk21TT>Es0_z2#DSnE8jrib1TjpG5x(?IS=z{FYMY6GZ*27;tYC4mv^gIyyX=exV!9FK;vSv>wdH~EL{7;2x;U6laI4k z@7fQ)`@X&Xmfy^yOU6NwhxfvITo!Q`BW!w@Y;#X;+L4Hi;*wGo!@|FIl$nq}dI|@6Bf2qC6&1ru1Om z5E-AURJ?x~V#kvua^}*hE64UQ7H%JW57 zghE54ur@T`JuPuf(p7sJ1-pQRtSaeM-Jvt2SfUD1uz&otP*<6WdqH;+eyhi__q|MG47 z<3Ifi-S4b@`s=^u-GtAXTxbdy2Rj)a{dXbR8`kd8chc;#6T?Kiy}Nwde)o_6f?+S( zr@#ENJ^S=I2IuR6Id$!X1jEzNT%*)GM&B7oQYTbBrxGsVI741BmUOb?Dn<;>^0Yf3 z)e7D4NGDACP#x+7sLEc0M^A;FOgW8EXb=Gkv@D72&OeA@7+AH&pnhCT*-_;cFXQql zD&^;Q!l{k+C5E+kq}vM}!c@_q15n`)q12ExG`yWaf_A; z%}QmtDT}KHWn-n(2vjr?ny#qU)fC<`;G+1!>mZ@Z1`o~0C6)^e=L$n~Yt0vy%Y#Rs z;;C%TCF8csvm0Jibkn~2>%VQEKIcbj7>jxI>W(vr3%c_qZ&qXw*Sj^!L1Is^6@ zlEY`5fd1r|lhhnM-~`MY9(KBK{QMW_yDYKPri~RuUJPvINgfh;VG~}~9Tw%1zYk)W zo9-1uo!0K8rB>0U)ejcJQ2%H{wGc}x`2h&(`8^%1h*B~6#-S8-4!MS2e%?H4lDys? zsRQY7s|HAZaC&UCN#qJ{wWBQ?PiT3g-7I#96Z+&8LFE|IJ>H=2qFza$j7X&UEY7f#E3obx=T0RBWz>J#97|NRg*)A}=bByo+ z&oraKX=)lK<8Iy_?zw<;Tx zq75cgKCM@E60d)Xq^E&~N9krguD(S5i@_Tegwcjx%R3s0Mv+Z~D#y5=NJ74<{c~ju zv4%_|9SR7uPFR&MmLe2#vHWw49~T)S*ofUJdZhqV7>vdhXnDT^SZ4knBn+phMig60 zi!!7f>64&|LFjZfb1$CBhpr4!4^lUbMtH$1+m^mma>Y+fFfOY(8eY)T%^daeUD5lT zUUmlTsp>o4LG5C*Z*I8e?BF%y#%toP_myb(Z6AKyjQM8ppW|^i=!*5?JJ4;iKDgje zZT+4xN54Ee<|Kt+FYKhm8`>E%tB@r#ad%p9+<0C*UaxV8{OanE)Otq;z(!*~;1%Uh z;1hz8Xz>Ap6~-VHtJGPS6jmTd5{086h4ho}HTq&ca0f9#7OtX$gE0P8wKbt}mw+R$9v&1TOfNJt>Y5IC7@b^*+wG(ZK`y#-Cw0$;kK|OU z#I)h+JdG#9YZ-!*DGc(@c{pgr9&c;!X>ukSu(JzX@n?zT+fP=_mNaX7nKykzojIW@*Z>?aOA95|5k4CRhH!InE*py zEUjd6*Ksl??{M?eT%eU#AI!SICP#Ue>c1@ca!J1fRODz(GYg?G2Lma~utcbd0eK^Y z>w0Adzfs~qT)$SeR75AQOv@+qOT~lm02xYBqG+<`7e`5bNAkgBmKc^37rm2PeClDwezNf;1^NmP zCcxPV$>CLu%kzWErDwEOo}`X(DxhiBB>nYHo6O&gGi^8TPR`;Q*~pMIRqVV#?ErPI z7K)dZPHat~9~xvx!=PA%9wON28^igT?-x6~?ZRI$tP=u$Uv7g-sk3K?IM_U*n% zkIQw{*iVlC3DgNCFN4`vVT&DD(80rOD1+i3jAOtWP=C!o&3b71`B#)>;P?s>}RvN{SiVKFg zj7dbvgAp1c)J%pKF9$E1GJv%b`3v{pm>yyr80jF zhv?A%1f@(zo1I)yDu`g6_+w1SMPENEWpq#gi{Me_D32~`l#u-Yr>iqa*kAnT{R{j}{-ltsh>ck9%9?{W*}VJq^SoKbnNX44U4;S?iOfVI z0VII0puzwNku|+W=e5ZQ?0l9zmfN zbbzLg^b-{Y&oFl^GdW8$5OH|DWcnMeaSy<0BSrOl1 znE__O6GKDpQ@y}0o#45%N4Q#4d(af+^C&$fn$Sd!@`aMdkUqQ(or*@%Oj=jzC1KH! zh!wYPM4y)q1l9YmTQPZJvK~s&KHLkx^Xz*WY4R?QVnqT7_BYt&b?^`_Nw9nXW=*p| z#7ksrm4G^6xeo}wge7!u|L*;lnpp1(S`opxMo&R(GlQ^57}q!@fD$!s>O6bpN6nFp zE9}c{;b|EJE<2KQ5J5PTp&?cr^ODlc`KtLgkL`~?{-u4#Cg`t!^MZ6t(V^|!9e?am zvo5QYDUii*Pi1EWhvaB42B~R>V@Em*qTT2jbgm=O+0-4j)|`Q&uuo|2{Y65fU2=Fg zXqaw{(AT)EO)#@K*ruMN;z0Q8S9nm1KR$%_=Qpe_U`xA0j*?B8pkm6{m%oTrq$G|1 z23hp4F2z?)hi?j|u)0XOHE!S=+KHUckq=p>rsTPY-6hJPd(xWR_#NK@#Ydr4gns-n z`52M^vI+Cvs9ym9*zY=^WAfN zA9}!f;j5}J#)+X1ky0j0HV!=Ebs`1i02_Od~e)@l7aWn(1KK9WiJ%YB#jyO=>%2CckzOc z0E@`|V{!;Q^MF`nWD4tf1nv3Ww7#htTz5ofani%pF-tXO%xJJXEACnQ2#Ql^7Ax?C z362V&<#3G9Jpdzs2t;%Rw}~`B~ckLfd~3u3ZAsliCrBb$C!*n0w32J4;Xl!Uwvx#r$4ic!U;Pid{48wgP)*B zL^wFi%DRXEaLClMNFRH!OB@%(};Or4JmMi9lR3H!6 zvA6w557^-#oAZh95b(^<@a*PBYSnj#Z|V3j0|P8=%@s9bz`%O@<)(f7(;wRRFJ8CN z;St)}v_-2D8Zkg%C1giLNt1hcAm#n8<}ld$wiunqic^6_@9>(0rZuzTy$@FX9)@rE z@YsQ=XQL&aNug7RY|rP4Uof*PS2D4^;409CLyMi#! z){5s~qjrc1@aqC9L496=P&fiR-zGHFAOB!R+BkD1p6>BB8x-87n>h6$CG z@Kk#Ek#OJ$%F7VLgyUb`3%Bs|S#Ka;bg3(jiM;~n3%s;ggbB|sTfP74+xRN5KC?hJ zP!-T6@Uc>weRr8A>Ani<;8xs9RY9QYW8#`M-a1=;b5m1Z8gMVIDg4B5Ri!0R3 z4n}V|Gi1$aN*i|JU(mFz=4_pY=Fzo&?LIFwyHg})9}T4Oe>i=^JTV&~(KDUUTRy)y z#sQTRo0uzxw+NC;r5Jdov!LKD5EM7X^t?O9s-u9EQ(FobFe&`@)~yqBgkG=cfgTIQ zd1U`X7fj_KM#@(4nRUTK`w@S`gS|tRlQK^X4EHB=$$$cA)4q+-gnp(+e)-`;d;Q(N zv;pgxd-DTo@dttkW3^h+sTY(q0+KCPQINMGZXYU&rD!}6yF^2|q0v4akw;dt&1N@kz*^x6 z%Q#1DxrV&bgo6Q^(0y!N;N(n-Qsos4EPDeM$C^mtWS)YeQW)q&ic{zs&9FSbA(Qz<9yXm^`Vk9@DMD^f8b=uC)&Jl=dd-I!D>8ZxPLpx46z{ZY7W-Dlx z-yCqh2s;^kzB+9qhG?d&@sPMq$=OsZL<)i$P9ldtlu18Ar{Pw7)Ra#BW!G06A>@U2 zpq(1Er+qThu9$;fq6J?aaN_$O3#-Bms7%>i%Mw8$Z_Do%#r?IszvPggS=&dW>-Cex zTJUQN%~hS@3h=*(l^G%!LG~-Y%BBD{MctWa6pTDJ8E@ntk5w6o)*mktIZ+6WWyFeI z&%%@cJXr~H=a(VUu3ptM-`NJl_!*lcx@eRjnu31_s|HFwNmhWm+vm8x*GV);0Z&1* zPDI%XRR&?}RPE^UkTBiVdk8Z;%K{-bEqY?xHMEyoV{VheF{_2X5Oor%7i2e%%4v%NGX` zlrseTmW5E-brZZURTQRk`KJ}px!v^g$=`z)Gqb|5iu=*5@vd>*@&GMt-BQT43EkOM zjk!g8+c|ml@*sO9x^8&~Ty%D=u}U;}2G9{D=;oo{xyW}9jlG_;?bn74J)yDek*2B4 zN_?2FfD#~mZ3JK$iO4O?DJm|<{o4Ik3Ls;-U^mm$Ek5Gn~3f>gOiJss@QLFIXi8)pZ~Y*z55=GyGMD@ z*uDp`jJX0MnvCaOqdsdN;FZ7>L^xr>ddNF(c@DtmHwiPy5)`bWm9;Z+V}`!-CHKAH zfNgwQ*M&kySACp7YA#oO#=7S z?$`%12O-qP+UPLAd2m~5^x~GqLpHBytjFuCw&c8EI}rMqZRu=K?GZNgBb|qhI~ZVZ zId&IKgg{tGPc8OrIqzd$fUjG&J>1?B&mQkCu30#ja_>&5nSulVU1l?8*Ebw`GvoL# z-0#)+6n}_x?E>#TGZYS7U*25dzUBUguKTr?gFboXAV-OcI&vZW5|aT5N=L|M-iqwr zf<$5L+u0O5KupIWd57*(!@WxZWSB!ac539A+(^sey#mRvVz|LJ)vleg9QEXn|J@FL zd)40m!|&RN69P67L?VJq&od#Fz>!*(3RuBT4#X`R0Sw{YZTjF3` zVcP<9@VKvZypwl*K?2D5m1_hxPxa*(D^9IE2qy^1NFoauK{7$`;a$8djHh6BNcxoK zy7GWhrUQm;5OHjL?Np@K$nOAw+G5kKZjRA|_TZ$5N1&r-`~fh`xk%M#&JCkcY03fE6DRDFfscRw19{Ec+aLa!vFig^Mg^gQR_6wsxDPA!$guW5U*1#5 zj>oz89sHJ+`Sx`xh9UA%oJuBqj@%g1%R&+wiyQN$+DMZ;8{gz!{A%kPJupchFPfVA zBG{N6B8g>C#WrmoG6cZG*O?o|i2A=(ua^+z+2IpT(cIz$y2hYhU7xfK=L(PRudt)o zG?I>mUdY)u8XHe1PTi<<236BIYk)-t(`nssS#fjAI%(7Va2h49TE_Vc*MB^56zmG7 zxaW(WDfCNVJg2H7Y}RI2TxBp3c@44m=G)!&S5Dph{SQa&-J5=ozr+MbPF^CbePJAO zYSNzeBx8NpqI$Y45NdnZLIxrif`n>w*hbA>tro_XrG7Y#3@BOU{2t4>H`4b@@tTbMH$BcUx zzr3q(6B-5a1At}r^JVThZ?(@%BVcZg*a0_f?o+jFSOx2Qu@_u<=H$KpUayZ0;P?RWq5wdyj2!W+qe#O}{g@<^rjJSWozSK?X0Q=4+8j;OVNty!+9{Z7aTeR!Pk0wA zD}seab7P`iDyPbQtro>BSfW|B^q4WoC_gd*aZ))k53JA|G$P_E!N37qm}d#=LEmn; zsV4N)P9kBLLQBRF9kkvu(7wd<5XOjKVb9oj{emH;9cE#yOfGuToX|UmDu}ttqWiOB zhLbpI3@y4<;3(6U3g!W3Yko%@U$*z^JqHUN!77J_DUW$WyO>JkCyi>!5t+hE@eQqb zRI?iw$*wT&6eL9!P|%)xrP2qHM)`cXl2P|4ZHQ-FX$BIWtTonVN~Rz}@ugF_qc zbl|xl3WhpJoZ@2#pw2Z9+tXGFzJJnB+`2;@{I0t3lkj+~&>m zheI?fj2)H`9<(Kbn;RXo0s7A$FF2ZkUHS3&*ZDvJv)E}?`E~Y0?WDGrF5>U|-U%U| zGBSYj_y&JjDie$)NV`y{1MR*saC4HA!2Bo;4)O8JAP<@r2BPU4s#);4U>)-XXC#jq z$X2V%D|w<;v~%Zi6LV}tPmuDA6E{-dju>#~Aioi~(p7598?%{pV>%R6wSK#>cTk1`W{qRXl*3I^p8e}yD& zYn0nOA~F(9mO&>4(6Qb+Ob7KZ-O*>2kROe1!y?)In9|JaPjCvT2tjPzmNu*=+c<`G z@u{sYKCs4uGd##p&*QZ=9SpK#AmfGOdSC}@CB6b|^OoZmfNGOx<lguKZwnV2u;YEU77Z-QZ zKDrY5GQ~g2c2b5Ng`<&tk$!iw4V0<>HLa)rrqGunv8;?$&7tUv4!jY(GeeU z7At=3q@-sKE2VNZmDYUhS}POz*<2}{6SS9|lQhr5pm~oiC=n>5T3-Rp4zN_%tDF1w zkxtXy9J`4gew<&sGLj&gam!Hw7RRy9yAj9Ea@vzNF(gPi9otMrCyyFSB%E>BEwpzW zlWGgC9Q*Q~iv5`x85GBqU0_EzOAOVnx{j&wMAHwni4cZM1SnX(kb$}2IS43!R$lX0 zC}RV5zXgoQJdb6*D&CeM!l=LjF!o;rPsoO;I?d3CG%skr0YfVTP9C?TK~?1WwoQR$ zjrE|Hg|!WBXCcV}IKn_MVUAi)#MHhDrq4UrGI}pNU~6+L3~ee$;GWnEL*{w1h;dZs zQrX!Au!wi@6#!}p>q7!n*%h8 z@kt;Rt$Y%htMmcTY-gd>h~L+(*MEjBQhZhuc%7T^wO}8Dq1;bh+4i2)J9!%KI1X{ZOSxRUFhqU5t{o{(}lmA`0 z{84q~kNDE%%X4=R4abh$J3f_WJB6%9SY|%21mc8E?%W z;tIccyl79CZ1i(?No~RCr->UuNgYDHwmf?TgS+2*Ahe8@_@17rO=j=crqe~`&SGeb zIOq%}FoR6(#i`k}xyXN)g+U~zqeDJ0-b&%TWejD|HeLxKzX*tO(swv4tr|s}aEnHe zVU|P<>hQ-XVQMB%#k4ZeL?bwN2ipkO<*@-D zsiifP&^6O^P&OFu1DKNvW6hJy2-_~UlLEL<_+hF?fSJ|~j`Z237wMp|gt13EU`S|d z6U0EZQ)`zu^Y-~{mh}!g7SemT{qWht3v% z?Pj9f3`bOe3A0e6lQu^fCwTIV_g}%!BX}mAqp|`s@I#8CSQJ3a0`o>l$k#a3C}&JW z1Oz}XNJOGR54G<+>1chA&3E1oa468XtyofN z7ktfj0}{{$8km#+{faWWPZUgg`EhDG4`tvJ9EpLeetX@5z*Db`X}# z7J!uLND@0vggs4N0hq2BFhwIN5Xc2EX-;H@S_Of%XeNfV2_47K4Pv9lT0eiwf`E;ptrmH1nQT%n*gDOB1<{Rf&9 zVxXlUsTFw}##!SfmW+rtQues_vT4&uj~yv_DTmJyQks7~rnR3GUcIHg92gHX$ipMS z6b`4_;)aIywh6x>{v|^gFD|dxJ&R5XvlT865IKi!6f!#~=yrasbmR>j?ZKEZ0cyv| z9A#Jurkc0x-|*2_xfWE)JLb=i*^2o1@~&O69{P!+FFcmQ23H|SSyOqr2TsmJMhL3& zCT*rN^5^Z0+Y`d@D;#@(`nkhq5n@0ptc@TD#49zhp1yLEP zFt~cmY=#Kh8~jF$^W?*gH8{e*>5FuW{irrqeQm!*%L zNSUSxrcDtM?Hm?kD2vl^`ukx`azfIaL#-$6oOy1Aa5Q?uj70UuYo~oGg-xBPa-l32`mIw-OQd)oz zNlNpMw5E0(%t7xZ&AiROd8an=)~PFp6T%9SrCb{({vWAqm-lEw(w^a3<^kI8-yC8e z5?{#^Nv`0{7c8ddipH0eX#BjRvr}a)A+&NtixL+Sg&u0iB{tp-8*(ayGkT=gbd1() zEuzzbRM9r;>0~`i@0u{@3M_w>S^Woluq$mgOZcQgH{g!^*yCM;Q+y`vAV4=OK+Ga* zqfKpoZCwB^Y~uLm3Pi{sc;f7WF3)^1f5~AhVyj{YOncV~xl6o-C)MLOaC32n&3Hs7 zhG8E3C`SzSAf3pLK*6!pqYNP5Xc6vF;WopQTjm?-m@Ht3q;@*1Hv5K<#Hj&P3+OnX zwU_3P%$H8%Ar!h^rL%${YU9Z)atU+zjxd|CD&JZfd(zOa)U=j@ZDYyJIfg z!xLw8M9!F=vom6kJsw3A$*tn-h@sa2uV%BStpGV6-Itjh!AY%W-es;xLo1q>t6r z-7PQ^o}km?NqZW8YHKzJoxJ|Gy?FN>Xkjy=2>u_?Y$)~^%tXxq0000 0: + ctx = [mx.gpu(i) for i in range(num_gpus)] + else: + ctx = [mx.cpu()] + return ctx + + +ALPHABET = '' +for i in range(27): + ALPHABET += int2char(i) + +def char_beam_search(out): + """ + Description : apply beam search for prediction result + """ + out_conv = list() + for idx in range(out.shape[0]): + probs = out[idx] + prob = probs.softmax().asnumpy() + line_string_proposals = ctcBeamSearch(prob, ALPHABET, None, k=4, beamWidth=25) + out_conv.append(line_string_proposals[0]) + return out_conv + +# pylint: disable=too-many-instance-attributes, too-many-locals +class Train: + """ + Description : Train class for training network + """ + def __init__(self, config): + ##setting hyper-parameters + self.batch_size = config.batch_size + self.image_path = config.image_path + self.align_path = config.align_path + self.num_gpus = config.num_gpus + self.ctx = setting_ctx(self.num_gpus) + self.num_workers = config.num_workers + self.seq_len = 75 + + def build_model(self, dr_rate=0, path=None): + """ + Description : build network + """ + #set network + self.net = LipNet(dr_rate) + self.net.hybridize() + self.net.initialize(ctx=self.ctx) + + if path is not None: + self.load_model(path) + + #set optimizer + self.loss_fn = gluon.loss.CTCLoss() + self.trainer = gluon.Trainer(self.net.collect_params(), \ + optimizer='SGD') + + def save_model(self, epoch, loss): + """ + Description : save parameter of network weight + """ + prefix = 'checkpoint/epoches' + file_name = "{prefix}_{epoch}_loss_{l:.4f}".format(prefix=prefix, + epoch=str(epoch), + l=loss) + self.net.save_parameters(file_name) + + def load_model(self, path=''): + """ + Description : load parameter of network weight + """ + self.net.load_parameters(path) + + def load_dataloader(self): + """ + Description : Setup the dataloader + """ + + input_transform = transforms.Compose([transforms.ToTensor(), \ + transforms.Normalize((0.7136, 0.4906, 0.3283), \ + (0.1138, 0.1078, 0.0917))]) + training_dataset = LipsDataset(self.image_path, + self.align_path, + mode='train', + transform=input_transform, + seq_len=self.seq_len) + + self.train_dataloader = mx.gluon.data.DataLoader(training_dataset, + batch_size=self.batch_size, + shuffle=True, + num_workers=self.num_workers) + + valid_dataset = LipsDataset(self.image_path, + self.align_path, + mode='valid', + transform=input_transform, + seq_len=self.seq_len) + + self.valid_dataloader = mx.gluon.data.DataLoader(valid_dataset, + batch_size=self.batch_size, + shuffle=True, + num_workers=self.num_workers) + + def train(self, data, label, batch_size): + """ + Description : training for LipNet + """ + # pylint: disable=no-member + sum_losses = 0 + len_losses = 0 + with autograd.record(): + losses = [self.loss_fn(self.net(X), Y) for X, Y in zip(data, label)] + for loss in losses: + sum_losses += mx.nd.array(loss).sum().asscalar() + len_losses += len(loss) + loss.backward() + self.trainer.step(batch_size) + return sum_losses, len_losses + + def infer(self, input_data, input_label): + """ + Description : Print sentence for prediction result + """ + sum_losses = 0 + len_losses = 0 + for data, label in zip(input_data, input_label): + pred = self.net(data) + sum_losses += mx.nd.array(self.loss_fn(pred, label)).sum().asscalar() + len_losses += len(data) + pred_convert = char_beam_search(pred) + label_convert = char_conv(label.asnumpy()) + for target, pred in zip(label_convert, pred_convert): + print("target:{t} pred:{p}".format(t=target, p=pred)) + return sum_losses, len_losses + + def train_batch(self, dataloader): + """ + Description : training for LipNet + """ + sum_losses = 0 + len_losses = 0 + for input_data, input_label in tqdm(dataloader): + data = gluon.utils.split_and_load(input_data, self.ctx, even_split=False) + label = gluon.utils.split_and_load(input_label, self.ctx, even_split=False) + batch_size = input_data.shape[0] + sum_losses, len_losses = self.train(data, label, batch_size) + sum_losses += sum_losses + len_losses += len_losses + + return sum_losses, len_losses + + def infer_batch(self, dataloader): + """ + Description : inference for LipNet + """ + sum_losses = 0 + len_losses = 0 + for input_data, input_label in dataloader: + data = gluon.utils.split_and_load(input_data, self.ctx, even_split=False) + label = gluon.utils.split_and_load(input_label, self.ctx, even_split=False) + sum_losses, len_losses = self.infer(data, label) + sum_losses += sum_losses + len_losses += len_losses + + return sum_losses, len_losses + + def run(self, epochs): + """ + Description : Run training for LipNet + """ + best_loss = sys.maxsize + for epoch in trange(epochs): + iter_no = 0 + + ## train + sum_losses, len_losses = self.train_batch(self.train_dataloader) + + if iter_no % 20 == 0: + current_loss = sum_losses / len_losses + print("[Train] epoch:{e} iter:{i} loss:{l:.4f}".format(e=epoch, + i=iter_no, + l=current_loss)) + + ## validating + sum_val_losses, len_val_losses = self.infer_batch(self.valid_dataloader) + + current_val_loss = sum_val_losses / len_val_losses + print("[Vaild] epoch:{e} iter:{i} loss:{l:.4f}".format(e=epoch, + i=iter_no, + l=current_val_loss)) + + if best_loss > current_val_loss: + self.save_model(epoch, current_val_loss) + best_loss = current_val_loss + + iter_no += 1 diff --git a/example/gluon/lipnet/utils/__init__.py b/example/gluon/lipnet/utils/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/example/gluon/lipnet/utils/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/example/gluon/lipnet/utils/align.py b/example/gluon/lipnet/utils/align.py new file mode 100644 index 000000000000..48d0716aaedd --- /dev/null +++ b/example/gluon/lipnet/utils/align.py @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Module: align +This is used when the data is genrated by LipsDataset +""" + +import numpy as np +from .common import word_to_vector + + +class Align(object): + """ + Preprocess for Align + """ + skip_list = ['sil', 'sp'] + + def __init__(self, align_path): + self.build(align_path) + + def build(self, align_path): + """ + Build the align array + """ + file = open(align_path, 'r') + lines = file.readlines() + file.close() + # words: list([op, ed, word]) + words = [] + for line in lines: + _op, _ed, word = line.strip().split(' ') + if word not in Align.skip_list: + words.append((int(_op), int(_ed), word)) + self.words = words + self.n_words = len(words) + self.sentence_str = " ".join([w[2] for w in self.words]) + self.sentence_length = len(self.sentence_str) + + def sentence(self, padding=75): + """ + Get sentence + """ + vec = word_to_vector(self.sentence_str) + vec += [-1] * (padding - self.sentence_length) + return np.array(vec, dtype=np.int32) + + def word(self, _id, padding=75): + """ + Get words + """ + word = self.words[_id][2] + vec = word_to_vector(word) + vec += [-1] * (padding - len(vec)) + return np.array(vec, dtype=np.int32) + + def word_length(self, _id): + """ + Get the length of words + """ + return len(self.words[_id][2]) + + def word_frame_pos(self, _id): + """ + Get the position of words + """ + left = int(self.words[_id][0]/1000) + right = max(left+1, int(self.words[_id][1]/1000)) + return (left, right) diff --git a/example/gluon/lipnet/utils/common.py b/example/gluon/lipnet/utils/common.py new file mode 100644 index 000000000000..ec96b6879653 --- /dev/null +++ b/example/gluon/lipnet/utils/common.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Module: This module contains common conversion functions + +""" + + +def char2int(char): + """ + Convert character to integer. + """ + if char >= 'a' and char <= 'z': + return ord(char) - ord('a') + elif char == ' ': + return 26 + return None + + +def int2char(num): + """ + Convert integer to character. + """ + if num >= 0 and num < 26: + return chr(num + ord('a')) + elif num == 26: + return ' ' + return None + + +def word_to_vector(word): + """ + Convert character vectors to integer vectors. + """ + vector = [] + for char in list(word): + vector.append(char2int(char)) + return vector + + +def vector_to_word(vector): + """ + Convert integer vectors to character vectors. + """ + word = "" + for vec in vector: + word = word + int2char(vec) + return word + + +def char_conv(out): + """ + Convert integer vectors to character vectors for batch. + """ + out_conv = list() + for i in range(out.shape[0]): + tmp_str = '' + for j in range(out.shape[1]): + if int(out[i][j]) >= 0: + tmp_char = int2char(int(out[i][j])) + if int(out[i][j]) == 27: + tmp_char = '' + tmp_str = tmp_str + tmp_char + out_conv.append(tmp_str) + return out_conv diff --git a/example/gluon/lipnet/utils/download_data.py b/example/gluon/lipnet/utils/download_data.py new file mode 100644 index 000000000000..3051eb2a9e27 --- /dev/null +++ b/example/gluon/lipnet/utils/download_data.py @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Module: download_data +This module provides utilities for downloading the datasets for training LipNet +""" + +import os +from os.path import exists +from multi import multi_p_run, put_worker + + +def download_mp4(from_idx, to_idx, _params): + """ + download mp4s + """ + succ = set() + fail = set() + for idx in range(from_idx, to_idx): + name = 's' + str(idx) + save_folder = '{src_path}/{nm}'.format(src_path=_params['src_path'], nm=name) + if idx == 0 or os.path.isdir(save_folder): + continue + script = "http://spandh.dcs.shef.ac.uk/gridcorpus/{nm}/video/{nm}.mpg_vcd.zip".format( \ + nm=name) + down_sc = 'cd {src_path} && curl {script} --output {nm}.mpg_vcd.zip && \ + unzip {nm}.mpg_vcd.zip'.format(script=script, + nm=name, + src_path=_params['src_path']) + try: + print(down_sc) + os.system(down_sc) + succ.add(idx) + except OSError as error: + print(error) + fail.add(idx) + return (succ, fail) + + +def download_align(from_idx, to_idx, _params): + """ + download aligns + """ + succ = set() + fail = set() + for idx in range(from_idx, to_idx): + name = 's' + str(idx) + if idx == 0: + continue + script = "http://spandh.dcs.shef.ac.uk/gridcorpus/{nm}/align/{nm}.tar".format(nm=name) + down_sc = 'cd {align_path} && wget {script} && \ + tar -xvf {nm}.tar'.format(script=script, + nm=name, + align_path=_params['align_path']) + try: + print(down_sc) + os.system(down_sc) + succ.add(idx) + except OSError as error: + print(error) + fail.add(idx) + return (succ, fail) + + +if __name__ == '__main__': + import argparse + PARSER = argparse.ArgumentParser() + PARSER.add_argument('--src_path', type=str, default='../data/mp4s') + PARSER.add_argument('--align_path', type=str, default='../data') + PARSER.add_argument('--n_process', type=int, default=1) + CONFIG = PARSER.parse_args() + PARAMS = {'src_path': CONFIG.src_path, 'align_path': CONFIG.align_path} + N_PROCESS = CONFIG.n_process + + if exists('./shape_predictor_68_face_landmarks.dat') is False: + os.system('wget http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2 && \ + bzip2 -d shape_predictor_68_face_landmarks.dat.bz2') + + os.makedirs('{src_path}'.format(src_path=PARAMS['src_path']), exist_ok=True) + os.makedirs('{align_path}'.format(align_path=PARAMS['align_path']), exist_ok=True) + + if N_PROCESS == 1: + RES = download_mp4(0, 35, PARAMS) + RES = download_align(0, 35, PARAMS) + else: + # download movie files + RES = multi_p_run(tot_num=35, _func=put_worker, worker=download_mp4, \ + params=PARAMS, n_process=N_PROCESS) + + # download align files + RES = multi_p_run(tot_num=35, _func=put_worker, worker=download_align, \ + params=PARAMS, n_process=N_PROCESS) + + os.system('rm -f {src_path}/*.zip && rm -f {src_path}/*/Thumbs.db'.format( \ + src_path=PARAMS['src_path'])) + os.system('rm -f {align_path}/*.tar && rm -f {align_path}/Thumbs.db'.format( \ + align_path=PARAMS['align_path'])) diff --git a/example/gluon/lipnet/utils/multi.py b/example/gluon/lipnet/utils/multi.py new file mode 100644 index 000000000000..ce545b572de6 --- /dev/null +++ b/example/gluon/lipnet/utils/multi.py @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Module: preprocess with multi-process +""" + + +def multi_p_run(tot_num, _func, worker, params, n_process): + """ + Run _func with multi-process using params. + """ + from multiprocessing import Process, Queue + out_q = Queue() + procs = [] + + split_num = split_seq(list(range(0, tot_num)), n_process) + + print(tot_num, ">>", split_num) + + split_len = len(split_num) + if n_process > split_len: + n_process = split_len + + for i in range(n_process): + _p = Process(target=_func, + args=(worker, split_num[i][0], split_num[i][1], + params, out_q)) + _p.daemon = True + procs.append(_p) + _p.start() + + try: + result = [] + for i in range(n_process): + result.append(out_q.get()) + for i in procs: + i.join() + except KeyboardInterrupt: + print('Killing all the children in the pool.') + for i in procs: + i.terminate() + i.join() + return -1 + + while not out_q.empty(): + print(out_q.get(block=False)) + + return result + + +def split_seq(sam_num, n_tile): + """ + Split the number(sam_num) into numbers by n_tile + """ + import math + print(sam_num) + print(n_tile) + start_num = sam_num[0::int(math.ceil(len(sam_num) / (n_tile)))] + end_num = start_num[1::] + end_num.append(len(sam_num)) + return [[i, j] for i, j in zip(start_num, end_num)] + + +def put_worker(func, from_idx, to_idx, params, out_q): + """ + put worker + """ + succ, fail = func(from_idx, to_idx, params) + return out_q.put({'succ': succ, 'fail': fail}) + + +def test_worker(from_idx, to_idx, params): + """ + the worker to test multi-process + """ + params = params + succ = set() + fail = set() + for idx in range(from_idx, to_idx): + try: + succ.add(idx) + except ValueError: + fail.add(idx) + return (succ, fail) + + +if __name__ == '__main__': + RES = multi_p_run(35, put_worker, test_worker, params={}, n_process=5) + print(RES) diff --git a/example/gluon/lipnet/utils/preprocess_data.py b/example/gluon/lipnet/utils/preprocess_data.py new file mode 100644 index 000000000000..a13fad88af7a --- /dev/null +++ b/example/gluon/lipnet/utils/preprocess_data.py @@ -0,0 +1,262 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Module: preprocess_data +Reference: https://github.com/rizkiarm/LipNet +""" + +# pylint: disable=too-many-locals, no-self-use, c-extension-no-member + +import os +import fnmatch +import errno +import numpy as np +from scipy import ndimage +from scipy.misc import imresize +from skimage import io +import skvideo.io +import dlib + +def mkdir_p(path): + """ + Make a directory + """ + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +def find_files(directory, pattern): + """ + Find files + """ + for root, _, files in os.walk(directory): + for basename in files: + if fnmatch.fnmatch(basename, pattern): + filename = os.path.join(root, basename) + yield filename + +class Video(object): + """ + Preprocess for Video + """ + def __init__(self, vtype='mouth', face_predictor_path=None): + if vtype == 'face' and face_predictor_path is None: + raise AttributeError('Face video need to be accompanied with face predictor') + self.face_predictor_path = face_predictor_path + self.vtype = vtype + self.face = None + self.mouth = None + self.data = None + self.length = None + + def from_frames(self, path): + """ + Read from frames + """ + frames_path = sorted([os.path.join(path, x) for x in os.listdir(path)]) + frames = [ndimage.imread(frame_path) for frame_path in frames_path] + self.handle_type(frames) + return self + + def from_video(self, path): + """ + Read from videos + """ + frames = self.get_video_frames(path) + self.handle_type(frames) + return self + + def from_array(self, frames): + """ + Read from array + """ + self.handle_type(frames) + return self + + def handle_type(self, frames): + """ + Config video types + """ + if self.vtype == 'mouth': + self.process_frames_mouth(frames) + elif self.vtype == 'face': + self.process_frames_face(frames) + else: + raise Exception('Video type not found') + + def process_frames_face(self, frames): + """ + Preprocess from frames using face detector + """ + detector = dlib.get_frontal_face_detector() + predictor = dlib.shape_predictor(self.face_predictor_path) + mouth_frames = self.get_frames_mouth(detector, predictor, frames) + self.face = np.array(frames) + self.mouth = np.array(mouth_frames) + if mouth_frames[0] is not None: + self.set_data(mouth_frames) + + def process_frames_mouth(self, frames): + """ + Preprocess from frames using mouth detector + """ + self.face = np.array(frames) + self.mouth = np.array(frames) + self.set_data(frames) + + def get_frames_mouth(self, detector, predictor, frames): + """ + Get frames using mouth crop + """ + mouth_width = 100 + mouth_height = 50 + horizontal_pad = 0.19 + normalize_ratio = None + mouth_frames = [] + for frame in frames: + dets = detector(frame, 1) + shape = None + for det in dets: + shape = predictor(frame, det) + i = -1 + if shape is None: # Detector doesn't detect face, just return None + return [None] + mouth_points = [] + for part in shape.parts(): + i += 1 + if i < 48: # Only take mouth region + continue + mouth_points.append((part.x, part.y)) + np_mouth_points = np.array(mouth_points) + + mouth_centroid = np.mean(np_mouth_points[:, -2:], axis=0) + + if normalize_ratio is None: + mouth_left = np.min(np_mouth_points[:, :-1]) * (1.0 - horizontal_pad) + mouth_right = np.max(np_mouth_points[:, :-1]) * (1.0 + horizontal_pad) + + normalize_ratio = mouth_width / float(mouth_right - mouth_left) + + new_img_shape = (int(frame.shape[0] * normalize_ratio), + int(frame.shape[1] * normalize_ratio)) + resized_img = imresize(frame, new_img_shape) + + mouth_centroid_norm = mouth_centroid * normalize_ratio + + mouth_l = int(mouth_centroid_norm[0] - mouth_width / 2) + mouth_r = int(mouth_centroid_norm[0] + mouth_width / 2) + mouth_t = int(mouth_centroid_norm[1] - mouth_height / 2) + mouth_b = int(mouth_centroid_norm[1] + mouth_height / 2) + + mouth_crop_image = resized_img[mouth_t:mouth_b, mouth_l:mouth_r] + + mouth_frames.append(mouth_crop_image) + return mouth_frames + + def get_video_frames(self, path): + """ + Get video frames + """ + videogen = skvideo.io.vreader(path) + frames = np.array([frame for frame in videogen]) + return frames + + def set_data(self, frames): + """ + Prepare the input of model + """ + data_frames = [] + for frame in frames: + #frame H x W x C + frame = frame.swapaxes(0, 1) # swap width and height to form format W x H x C + if len(frame.shape) < 3: + frame = np.array([frame]).swapaxes(0, 2).swapaxes(0, 1) # Add grayscale channel + data_frames.append(frame) + frames_n = len(data_frames) + data_frames = np.array(data_frames) # T x W x H x C + data_frames = np.rollaxis(data_frames, 3) # C x T x W x H + data_frames = data_frames.swapaxes(2, 3) # C x T x H x W = NCDHW + + self.data = data_frames + self.length = frames_n + +def preprocess(from_idx, to_idx, _params): + """ + Preprocess: Convert a video into the mouth images + """ + source_exts = '*.mpg' + src_path = _params['src_path'] + tgt_path = _params['tgt_path'] + face_predictor_path = './shape_predictor_68_face_landmarks.dat' + + succ = set() + fail = set() + for idx in range(from_idx, to_idx): + s_id = 's' + str(idx) + '/' + source_path = src_path + '/' + s_id + target_path = tgt_path + '/' + s_id + fail_cnt = 0 + for filepath in find_files(source_path, source_exts): + print("Processing: {}".format(filepath)) + filepath_wo_ext = os.path.splitext(filepath)[0].split('/')[-2:] + target_dir = os.path.join(tgt_path, '/'.join(filepath_wo_ext)) + + if os.path.exists(target_dir): + continue + + try: + video = Video(vtype='face', \ + face_predictor_path=face_predictor_path).from_video(filepath) + mkdir_p(target_dir) + i = 0 + if video.mouth[0] is None: + continue + for frame in video.mouth: + io.imsave(os.path.join(target_dir, "mouth_{0:03d}.png".format(i)), frame) + i += 1 + except ValueError as error: + print(error) + fail_cnt += 1 + if fail_cnt == 0: + succ.add(idx) + else: + fail.add(idx) + return (succ, fail) + +if __name__ == '__main__': + import argparse + from multi import multi_p_run, put_worker + PARSER = argparse.ArgumentParser() + PARSER.add_argument('--src_path', type=str, default='../data/mp4s') + PARSER.add_argument('--tgt_path', type=str, default='../data/datasets') + PARSER.add_argument('--n_process', type=int, default=1) + CONFIG = PARSER.parse_args() + N_PROCESS = CONFIG.n_process + PARAMS = {'src_path':CONFIG.src_path, + 'tgt_path':CONFIG.tgt_path} + + os.makedirs('{tgt_path}'.format(tgt_path=PARAMS['tgt_path']), exist_ok=True) + + if N_PROCESS == 1: + RES = preprocess(0, 35, PARAMS) + else: + RES = multi_p_run(35, put_worker, preprocess, PARAMS, N_PROCESS) diff --git a/example/gluon/lipnet/utils/run_preprocess.ipynb b/example/gluon/lipnet/utils/run_preprocess.ipynb new file mode 100644 index 000000000000..7a25e9b33517 --- /dev/null +++ b/example/gluon/lipnet/utils/run_preprocess.ipynb @@ -0,0 +1,194 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "from download_data import multi_p_run, put_worker, _worker, download_mp4, download_align" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TEST" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]\n", + "5\n", + "35 >> [[0, 7], [7, 14], [14, 21], [21, 28], [28, 35]]\n", + "[{'succ': {0, 1, 2, 3, 4, 5, 6}, 'fail': set()}, {'succ': {7, 8, 9, 10, 11, 12, 13}, 'fail': set()}, {'succ': {14, 15, 16, 17, 18, 19, 20}, 'fail': set()}, {'succ': {21, 22, 23, 24, 25, 26, 27}, 'fail': set()}, {'succ': {32, 33, 34, 28, 29, 30, 31}, 'fail': set()}]\n" + ] + } + ], + "source": [ + "res = multi_p_run(35, put_worker, _worker, 5)\n", + "print (res)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## down\n", + "import os\n", + "os.makedirs('./datasets', exist_ok=True)\n", + "#os.system('rm -rf ./datasets/*')\n", + "\n", + "res = multi_p_run(35, put_worker, download_align, 9)\n", + "print (res)\n", + "\n", + "os.system('rm -f datasets/*.tar && rm -f datasets/align/Thumbs.db')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "res = multi_p_run(35, put_worker, download_mp4, 9)\n", + "print (res)\n", + "\n", + "os.system('rm -f datasets/*.zip && rm -f datasets/*/Thumbs.db')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## download single 22 th dir\n", + "#download_data.py(22, 22)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Preprocess Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from preprocess_data import preprocess, find_files, Video" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "os.makedirs('./TARGET', exist_ok=True)\n", + "os.system('rm -rf ./TARGET/*')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]\n", + "9\n", + "35 >> [[0, 4], [4, 8], [8, 12], [12, 16], [16, 20], [20, 24], [24, 28], [28, 32], [32, 35]]\n", + "Processing: datasets/s1/prwq3s.mpg\n", + "Processing: datasets/s4/lrix7n.mpg\n", + "Processing: datasets/s8/pgbyza.mpg\n", + "Processing: datasets/s12/brik7n.mpg\n", + "Processing: datasets/s16/sgit7p.mpg\n", + "Processing: datasets/s20/lrbp8a.mpg\n", + "Processing: datasets/s24/sbik8a.mpg\n", + "Processing: datasets/s28/srwf8a.mpg\n", + "Processing: datasets/s32/pbbm1n.mpg\n", + "Processing: datasets/s12/sbbaza.mpg\n", + "Processing: datasets/s28/lbit7n.mpg\n", + "Processing: datasets/s32/pbwm7p.mpg\n", + "Processing: datasets/s8/bril2s.mpg\n", + "Processing: datasets/s20/bway7n.mpg\n", + "Processing: datasets/s1/pbib8p.mpg\n", + "Processing: datasets/s16/lwaj7n.mpg\n", + "Processing: datasets/s24/bwwl6a.mpg\n", + "Processing: datasets/s4/bbwf7n.mpg\n" + ] + } + ], + "source": [ + "res = multi_p_run(35, put_worker, preprocess, 9)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/example/gluon/lipnet/utils/run_preprocess_single_process.ipynb b/example/gluon/lipnet/utils/run_preprocess_single_process.ipynb new file mode 100644 index 000000000000..4311323206e1 --- /dev/null +++ b/example/gluon/lipnet/utils/run_preprocess_single_process.ipynb @@ -0,0 +1,360 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "from download_data import multi_p_run, put_worker, test_worker, download_mp4, download_align" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "tot_movies=35" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TEST" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]\n", + "5\n", + "35 >> [[0, 7], [7, 14], [14, 21], [21, 28], [28, 35]]\n", + "[{'succ': {0, 1, 2, 3, 4, 5, 6}, 'fail': set()}, {'succ': {7, 8, 9, 10, 11, 12, 13}, 'fail': set()}, {'succ': {14, 15, 16, 17, 18, 19, 20}, 'fail': set()}, {'succ': {21, 22, 23, 24, 25, 26, 27}, 'fail': set()}, {'succ': {32, 33, 34, 28, 29, 30, 31}, 'fail': set()}]\n" + ] + } + ], + "source": [ + "res = multi_p_run(tot_movies, put_worker, test_worker, params={}, n_process=5)\n", + "print (res)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Aligns" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s0/align/s0.tar && tar -xvf s0.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s1/align/s1.tar && tar -xvf s1.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s2/align/s2.tar && tar -xvf s2.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s3/align/s3.tar && tar -xvf s3.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s4/align/s4.tar && tar -xvf s4.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s5/align/s5.tar && tar -xvf s5.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s6/align/s6.tar && tar -xvf s6.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s7/align/s7.tar && tar -xvf s7.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s8/align/s8.tar && tar -xvf s8.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s9/align/s9.tar && tar -xvf s9.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s10/align/s10.tar && tar -xvf s10.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s11/align/s11.tar && tar -xvf s11.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s12/align/s12.tar && tar -xvf s12.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s13/align/s13.tar && tar -xvf s13.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s14/align/s14.tar && tar -xvf s14.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s15/align/s15.tar && tar -xvf s15.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s16/align/s16.tar && tar -xvf s16.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s17/align/s17.tar && tar -xvf s17.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s18/align/s18.tar && tar -xvf s18.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s19/align/s19.tar && tar -xvf s19.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s20/align/s20.tar && tar -xvf s20.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s21/align/s21.tar && tar -xvf s21.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s22/align/s22.tar && tar -xvf s22.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s23/align/s23.tar && tar -xvf s23.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s24/align/s24.tar && tar -xvf s24.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s25/align/s25.tar && tar -xvf s25.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s26/align/s26.tar && tar -xvf s26.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s27/align/s27.tar && tar -xvf s27.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s28/align/s28.tar && tar -xvf s28.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s29/align/s29.tar && tar -xvf s29.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s30/align/s30.tar && tar -xvf s30.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s31/align/s31.tar && tar -xvf s31.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s32/align/s32.tar && tar -xvf s32.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s33/align/s33.tar && tar -xvf s33.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s34/align/s34.tar && tar -xvf s34.tar\n" + ] + } + ], + "source": [ + "align_path = '../data/align'\n", + "os.makedirs(align_path, exist_ok=True)\n", + "\n", + "res = download_align(0, tot_movies, {'align_path':align_path})" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34}, set())\n" + ] + }, + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print (res)\n", + "os.system('rm -f {align_path}/*.tar && rm -f {align_path}/Thumbs.db'.format(align_path=align_path))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "### Moives(MP4s)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s0/video/s0.mpg_vcd.zip --output s0.mpg_vcd.zip && unzip s0.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s1/video/s1.mpg_vcd.zip --output s1.mpg_vcd.zip && unzip s1.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s2/video/s2.mpg_vcd.zip --output s2.mpg_vcd.zip && unzip s2.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s3/video/s3.mpg_vcd.zip --output s3.mpg_vcd.zip && unzip s3.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s4/video/s4.mpg_vcd.zip --output s4.mpg_vcd.zip && unzip s4.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s5/video/s5.mpg_vcd.zip --output s5.mpg_vcd.zip && unzip s5.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s6/video/s6.mpg_vcd.zip --output s6.mpg_vcd.zip && unzip s6.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s7/video/s7.mpg_vcd.zip --output s7.mpg_vcd.zip && unzip s7.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s8/video/s8.mpg_vcd.zip --output s8.mpg_vcd.zip && unzip s8.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s9/video/s9.mpg_vcd.zip --output s9.mpg_vcd.zip && unzip s9.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s10/video/s10.mpg_vcd.zip --output s10.mpg_vcd.zip && unzip s10.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s11/video/s11.mpg_vcd.zip --output s11.mpg_vcd.zip && unzip s11.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s12/video/s12.mpg_vcd.zip --output s12.mpg_vcd.zip && unzip s12.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s13/video/s13.mpg_vcd.zip --output s13.mpg_vcd.zip && unzip s13.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s14/video/s14.mpg_vcd.zip --output s14.mpg_vcd.zip && unzip s14.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s15/video/s15.mpg_vcd.zip --output s15.mpg_vcd.zip && unzip s15.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s16/video/s16.mpg_vcd.zip --output s16.mpg_vcd.zip && unzip s16.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s17/video/s17.mpg_vcd.zip --output s17.mpg_vcd.zip && unzip s17.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s18/video/s18.mpg_vcd.zip --output s18.mpg_vcd.zip && unzip s18.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s19/video/s19.mpg_vcd.zip --output s19.mpg_vcd.zip && unzip s19.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s20/video/s20.mpg_vcd.zip --output s20.mpg_vcd.zip && unzip s20.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s21/video/s21.mpg_vcd.zip --output s21.mpg_vcd.zip && unzip s21.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s22/video/s22.mpg_vcd.zip --output s22.mpg_vcd.zip && unzip s22.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s23/video/s23.mpg_vcd.zip --output s23.mpg_vcd.zip && unzip s23.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s24/video/s24.mpg_vcd.zip --output s24.mpg_vcd.zip && unzip s24.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s25/video/s25.mpg_vcd.zip --output s25.mpg_vcd.zip && unzip s25.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s26/video/s26.mpg_vcd.zip --output s26.mpg_vcd.zip && unzip s26.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s27/video/s27.mpg_vcd.zip --output s27.mpg_vcd.zip && unzip s27.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s28/video/s28.mpg_vcd.zip --output s28.mpg_vcd.zip && unzip s28.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s29/video/s29.mpg_vcd.zip --output s29.mpg_vcd.zip && unzip s29.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s30/video/s30.mpg_vcd.zip --output s30.mpg_vcd.zip && unzip s30.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s31/video/s31.mpg_vcd.zip --output s31.mpg_vcd.zip && unzip s31.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s32/video/s32.mpg_vcd.zip --output s32.mpg_vcd.zip && unzip s32.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s33/video/s33.mpg_vcd.zip --output s33.mpg_vcd.zip && unzip s33.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s34/video/s34.mpg_vcd.zip --output s34.mpg_vcd.zip && unzip s34.mpg_vcd.zip\n" + ] + } + ], + "source": [ + "src_path = '../data/mp4s'\n", + "res = download_mp4(0, tot_movies, {'src_path':src_path})" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34}, set())\n" + ] + }, + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print (res)\n", + "os.system('rm -f {src_path}/*.zip && rm -f {src_path}/*/Thumbs.db'.format(src_path=src_path))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Preprocess Data" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from preprocess_data import preprocess, find_files, Video" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "tgt_path = '../data/datasets'" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "os.makedirs('{tgt_path}'.format(tgt_path=tgt_path), exist_ok=True)\n", + "os.system('rm -rf {tgt_path}'.format(tgt_path=tgt_path))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "res = preprocess(0, tot_movies, {'src_path':src_path, 'tgt_path':tgt_path})" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34}, set())\n" + ] + } + ], + "source": [ + "print (res)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [default]", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 0b1761ff118e4724a9d934aa018de90acadda17f Mon Sep 17 00:00:00 2001 From: Yuxi Hu Date: Wed, 13 Feb 2019 13:36:28 -0800 Subject: [PATCH 13/75] Add pin_device_id option to Gluon DataLoader (#14136) * add pin_device_id option to DataLoader * add unit test to check output context * trigger CI --- python/mxnet/gluon/data/dataloader.py | 37 +++++++++++++++--------- tests/python/unittest/test_gluon_data.py | 24 +++++++++++++++ 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py index 9d762745a407..934f2d5954c1 100644 --- a/python/mxnet/gluon/data/dataloader.py +++ b/python/mxnet/gluon/data/dataloader.py @@ -169,14 +169,15 @@ def worker_loop_v1(dataset, key_queue, data_queue, batchify_fn): batch = batchify_fn([dataset[i] for i in samples]) data_queue.put((idx, batch)) -def fetcher_loop_v1(data_queue, data_buffer, pin_memory=False, data_buffer_lock=None): +def fetcher_loop_v1(data_queue, data_buffer, pin_memory=False, + pin_device_id=0, data_buffer_lock=None): """Fetcher loop for fetching data from queue and put in reorder dict.""" while True: idx, batch = data_queue.get() if idx is None: break if pin_memory: - batch = _as_in_context(batch, context.cpu_pinned()) + batch = _as_in_context(batch, context.cpu_pinned(pin_device_id)) else: batch = _as_in_context(batch, context.cpu()) if data_buffer_lock is not None: @@ -188,8 +189,8 @@ def fetcher_loop_v1(data_queue, data_buffer, pin_memory=False, data_buffer_lock= class _MultiWorkerIterV1(object): """Internal multi-worker iterator for DataLoader.""" - def __init__(self, num_workers, dataset, batchify_fn, batch_sampler, pin_memory=False, - worker_fn=worker_loop_v1): + def __init__(self, num_workers, dataset, batchify_fn, batch_sampler, + pin_memory=False, pin_device_id=0, worker_fn=worker_loop_v1): assert num_workers > 0, "_MultiWorkerIter is not for {} workers".format(num_workers) self._num_workers = num_workers self._dataset = dataset @@ -218,7 +219,8 @@ def __init__(self, num_workers, dataset, batchify_fn, batch_sampler, pin_memory= self._fetcher = threading.Thread( target=fetcher_loop_v1, - args=(self._data_queue, self._data_buffer, pin_memory, self._data_buffer_lock)) + args=(self._data_queue, self._data_buffer, pin_memory, + pin_device_id, self._data_buffer_lock)) self._fetcher.daemon = True self._fetcher.start() @@ -323,12 +325,15 @@ def default_batchify_fn(data): If ``True``, the dataloader will copy NDArrays into pinned memory before returning them. Copying from CPU pinned memory to GPU is faster than from normal CPU memory. + pin_device_id : int, default 0 + The device id to use for allocating pinned memory if pin_memory is ``True`` """ def __init__(self, dataset, batch_size=None, shuffle=False, sampler=None, last_batch=None, batch_sampler=None, batchify_fn=None, - num_workers=0, pin_memory=False): + num_workers=0, pin_memory=False, pin_device_id=0): self._dataset = dataset self._pin_memory = pin_memory + self._pin_device_id = pin_device_id if batch_sampler is None: if batch_size is None: @@ -365,13 +370,14 @@ def same_process_iter(): for batch in self._batch_sampler: ret = self._batchify_fn([self._dataset[idx] for idx in batch]) if self._pin_memory: - ret = _as_in_context(ret, context.cpu_pinned()) + ret = _as_in_context(ret, context.cpu_pinned(self._pin_device_id)) yield ret return same_process_iter() # multi-worker return _MultiWorkerIterV1(self._num_workers, self._dataset, - self._batchify_fn, self._batch_sampler, self._pin_memory) + self._batchify_fn, self._batch_sampler, + self._pin_memory, self._pin_device_id) def __len__(self): return len(self._batch_sampler) @@ -403,7 +409,7 @@ def _thread_worker_fn(samples, batchify_fn, dataset): class _MultiWorkerIter(object): """Internal multi-worker iterator for DataLoader.""" def __init__(self, worker_pool, batchify_fn, batch_sampler, pin_memory=False, - worker_fn=_worker_fn, prefetch=0, dataset=None): + pin_device_id=0, worker_fn=_worker_fn, prefetch=0, dataset=None): self._worker_pool = worker_pool self._batchify_fn = batchify_fn self._batch_sampler = batch_sampler @@ -413,6 +419,7 @@ def __init__(self, worker_pool, batchify_fn, batch_sampler, pin_memory=False, self._iter = iter(self._batch_sampler) self._worker_fn = worker_fn self._pin_memory = pin_memory + self._pin_device_id = pin_device_id self._dataset = dataset # pre-fetch for _ in range(prefetch): @@ -442,7 +449,7 @@ def __next__(self): ret = self._data_buffer.pop(self._rcvd_idx) batch = pickle.loads(ret.get()) if self._dataset is None else ret.get() if self._pin_memory: - batch = _as_in_context(batch, context.cpu_pinned()) + batch = _as_in_context(batch, context.cpu_pinned(self._pin_device_id)) batch = batch[0] if len(batch) == 1 else batch self._rcvd_idx += 1 return batch @@ -498,6 +505,8 @@ def default_batchify_fn(data): If ``True``, the dataloader will copy NDArrays into pinned memory before returning them. Copying from CPU pinned memory to GPU is faster than from normal CPU memory. + pin_device_id : int, default 0 + The device id to use for allocating pinned memory if pin_memory is ``True`` prefetch : int, default is `num_workers * 2` The number of prefetching batches only works if `num_workers` > 0. If `prefetch` > 0, it allow worker process to prefetch certain batches before @@ -514,9 +523,11 @@ def default_batchify_fn(data): """ def __init__(self, dataset, batch_size=None, shuffle=False, sampler=None, last_batch=None, batch_sampler=None, batchify_fn=None, - num_workers=0, pin_memory=False, prefetch=None, thread_pool=False): + num_workers=0, pin_memory=False, pin_device_id=0, + prefetch=None, thread_pool=False): self._dataset = dataset self._pin_memory = pin_memory + self._pin_device_id = pin_device_id self._thread_pool = thread_pool if batch_sampler is None: @@ -562,13 +573,13 @@ def same_process_iter(): for batch in self._batch_sampler: ret = self._batchify_fn([self._dataset[idx] for idx in batch]) if self._pin_memory: - ret = _as_in_context(ret, context.cpu_pinned()) + ret = _as_in_context(ret, context.cpu_pinned(self._pin_device_id)) yield ret return same_process_iter() # multi-worker return _MultiWorkerIter(self._worker_pool, self._batchify_fn, self._batch_sampler, - pin_memory=self._pin_memory, + pin_memory=self._pin_memory, pin_device_id=self._pin_device_id, worker_fn=_thread_worker_fn if self._thread_pool else _worker_fn, prefetch=self._prefetch, dataset=self._dataset if self._thread_pool else None) diff --git a/tests/python/unittest/test_gluon_data.py b/tests/python/unittest/test_gluon_data.py index 353a819ddbf6..1939de82eb44 100644 --- a/tests/python/unittest/test_gluon_data.py +++ b/tests/python/unittest/test_gluon_data.py @@ -256,6 +256,30 @@ def test_multi_worker_dataloader_release_pool(): del the_iter del D + +def test_dataloader_context(): + X = np.random.uniform(size=(10, 20)) + dataset = gluon.data.ArrayDataset(X) + default_dev_id = 0 + custom_dev_id = 1 + + # use non-pinned memory + loader1 = gluon.data.DataLoader(dataset, 8) + for _, x in enumerate(loader1): + assert x.context == context.cpu(default_dev_id) + + # use pinned memory with default device id + loader2 = gluon.data.DataLoader(dataset, 8, pin_memory=True) + for _, x in enumerate(loader2): + assert x.context == context.cpu_pinned(default_dev_id) + + # use pinned memory with custom device id + loader3 = gluon.data.DataLoader(dataset, 8, pin_memory=True, + pin_device_id=custom_dev_id) + for _, x in enumerate(loader3): + assert x.context == context.cpu_pinned(custom_dev_id) + + if __name__ == '__main__': import nose nose.runmodule() From 85d3fa34901c8c31815aa59ae5e125e3c6feea9b Mon Sep 17 00:00:00 2001 From: Carin Meier Date: Wed, 13 Feb 2019 18:44:55 -0500 Subject: [PATCH 14/75] upgrade codox to work with lein 2.9.0 (#14133) --- contrib/clojure-package/project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/clojure-package/project.clj b/contrib/clojure-package/project.clj index 61d39e28a1d6..e2b999df84d6 100644 --- a/contrib/clojure-package/project.clj +++ b/contrib/clojure-package/project.clj @@ -36,7 +36,7 @@ [org.apache.logging.log4j/log4j-api "2.8.1"] [org.slf4j/slf4j-log4j12 "1.7.25" :exclusions [org.slf4j/slf4j-api]]] :pedantic? :skip - :plugins [[lein-codox "0.10.3" :exclusions [org.clojure/clojure]] + :plugins [[lein-codox "0.10.6" :exclusions [org.clojure/clojure]] [lein-cloverage "1.0.10" :exclusions [org.clojure/clojure]] [lein-cljfmt "0.5.7"]] :codox {:namespaces [#"^org\.apache\.clojure-mxnet\.(?!gen).*"]} From 6f9a67901362a794e3c022dd75daf8a516760fea Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Wed, 13 Feb 2019 16:24:22 -0800 Subject: [PATCH 15/75] fix website build (#14148) * fix rendering of markdown doc * fix missing license from lipnet example --- CONTRIBUTORS.md | 30 ++++++++-------- MKLDNN_README.md | 30 ++++++++-------- NEWS.md | 34 +++++++++---------- README.md | 34 +++++++++---------- amalgamation/README.md | 34 +++++++++---------- ci/README.md | 34 +++++++++---------- ci/docker/qemu/README.md | 30 ++++++++-------- ci/publish/README.md | 34 +++++++++---------- ci/qemu/README.md | 34 +++++++++---------- contrib/clojure-package/README.md | 34 +++++++++---------- .../examples/captcha/README.md | 30 ++++++++-------- .../cnn-text-classification/README.md | 34 +++++++++---------- .../clojure-package/examples/gan/README.md | 34 +++++++++---------- .../examples/imclassification/README.md | 34 +++++++++---------- .../examples/infer/imageclassifier/README.md | 34 +++++++++---------- .../examples/infer/objectdetector/README.md | 34 +++++++++---------- .../examples/infer/predictor/README.md | 34 +++++++++---------- .../clojure-package/examples/module/README.md | 34 +++++++++---------- .../examples/multi-label/README.md | 34 +++++++++---------- .../examples/neural-style/README.md | 34 +++++++++---------- .../examples/pre-trained-models/README.md | 34 +++++++++---------- .../examples/profiler/README.md | 30 ++++++++-------- .../clojure-package/examples/rnn/README.md | 34 +++++++++---------- .../examples/tutorial/README.md | 30 ++++++++-------- .../examples/visualization/README.md | 30 ++++++++-------- contrib/clojure-package/testing.md | 34 +++++++++---------- cpp-package/README.md | 34 +++++++++---------- cpp-package/example/README.md | 34 +++++++++---------- cpp-package/example/feature_extract/README.md | 30 ++++++++-------- cpp-package/example/inference/README.md | 34 +++++++++---------- docs/README.md | 34 +++++++++---------- docs/api/c++/index.md | 34 +++++++++---------- docs/api/clojure/index.md | 34 +++++++++---------- docs/api/clojure/kvstore.md | 34 +++++++++---------- docs/api/clojure/module.md | 34 +++++++++---------- docs/api/clojure/ndarray.md | 34 +++++++++---------- docs/api/clojure/symbol.md | 34 +++++++++---------- docs/api/clojure/symbol_in_pictures.md | 34 +++++++++---------- docs/api/index.md | 30 ++++++++-------- docs/api/java/index.md | 34 +++++++++---------- docs/api/julia/index.md | 34 +++++++++---------- docs/api/perl/index.md | 34 +++++++++---------- docs/api/perl/io.md | 34 +++++++++---------- docs/api/perl/kvstore.md | 34 +++++++++---------- docs/api/perl/module.md | 34 +++++++++---------- docs/api/perl/ndarray.md | 34 +++++++++---------- docs/api/perl/symbol.md | 34 +++++++++---------- docs/api/python/autograd/autograd.md | 34 +++++++++---------- docs/api/python/callback/callback.md | 34 +++++++++---------- docs/api/python/contrib/contrib.md | 30 ++++++++-------- docs/api/python/contrib/onnx.md | 34 +++++++++---------- docs/api/python/contrib/svrg_optimization.md | 34 +++++++++---------- docs/api/python/contrib/text.md | 34 +++++++++---------- docs/api/python/executor/executor.md | 34 +++++++++---------- docs/api/python/gluon/contrib.md | 34 +++++++++---------- docs/api/python/gluon/data.md | 34 +++++++++---------- docs/api/python/gluon/gluon.md | 34 +++++++++---------- docs/api/python/gluon/loss.md | 34 +++++++++---------- docs/api/python/gluon/model_zoo.md | 34 +++++++++---------- docs/api/python/gluon/nn.md | 34 +++++++++---------- docs/api/python/gluon/rnn.md | 34 +++++++++---------- docs/api/python/image/image.md | 34 +++++++++---------- docs/api/python/index.md | 34 +++++++++---------- docs/api/python/io/io.md | 34 +++++++++---------- docs/api/python/kvstore/kvstore.md | 34 +++++++++---------- docs/api/python/metric/metric.md | 34 +++++++++---------- docs/api/python/model.md | 34 +++++++++---------- docs/api/python/module/module.md | 34 +++++++++---------- docs/api/python/ndarray/contrib.md | 34 +++++++++---------- docs/api/python/ndarray/linalg.md | 34 +++++++++---------- docs/api/python/ndarray/ndarray.md | 34 +++++++++---------- docs/api/python/ndarray/random.md | 34 +++++++++---------- docs/api/python/ndarray/sparse.md | 34 +++++++++---------- docs/api/python/optimization/contrib.md | 34 +++++++++---------- docs/api/python/optimization/optimization.md | 34 +++++++++---------- docs/api/python/rtc/rtc.md | 34 +++++++++---------- docs/api/python/symbol/contrib.md | 34 +++++++++---------- docs/api/python/symbol/linalg.md | 34 +++++++++---------- docs/api/python/symbol/random.md | 34 +++++++++---------- docs/api/python/symbol/rnn.md | 34 +++++++++---------- docs/api/python/symbol/sparse.md | 34 +++++++++---------- docs/api/python/symbol/symbol.md | 34 +++++++++---------- .../symbol_in_pictures/symbol_in_pictures.md | 34 +++++++++---------- docs/api/python/tools/test_utils.md | 34 +++++++++---------- docs/api/python/tools/visualization.md | 34 +++++++++---------- docs/api/r/index.md | 34 +++++++++---------- docs/api/scala/index.md | 34 +++++++++---------- docs/api/scala/infer.md | 34 +++++++++---------- docs/api/scala/io.md | 34 +++++++++---------- docs/api/scala/kvstore.md | 34 +++++++++---------- docs/api/scala/model.md | 34 +++++++++---------- docs/api/scala/module.md | 34 +++++++++---------- docs/api/scala/ndarray.md | 34 +++++++++---------- docs/api/scala/symbol.md | 34 +++++++++---------- docs/api/scala/symbol_in_pictures.md | 34 +++++++++---------- docs/architecture/exception_handling.md | 34 +++++++++---------- docs/architecture/index.md | 34 +++++++++---------- docs/architecture/note_data_loading.md | 34 +++++++++---------- docs/architecture/note_engine.md | 34 +++++++++---------- docs/architecture/note_memory.md | 34 +++++++++---------- docs/architecture/overview.md | 34 +++++++++---------- docs/architecture/program_model.md | 34 +++++++++---------- docs/architecture/rnn_interface.md | 34 +++++++++---------- docs/build_version_doc/README.md | 34 +++++++++---------- docs/community/contribute.md | 34 +++++++++---------- docs/community/ecosystem.md | 34 +++++++++---------- docs/community/index.md | 30 ++++++++-------- docs/community/mxnet_channels.md | 30 ++++++++-------- docs/community/powered_by.md | 30 ++++++++-------- docs/faq/add_op_in_backend.md | 34 +++++++++---------- docs/faq/bucketing.md | 34 +++++++++---------- docs/faq/caffe.md | 34 +++++++++---------- docs/faq/cloud.md | 30 ++++++++-------- docs/faq/develop_and_hack.md | 30 ++++++++-------- docs/faq/distributed_training.md | 34 +++++++++---------- docs/faq/env_var.md | 34 +++++++++---------- docs/faq/faq.md | 34 +++++++++---------- docs/faq/finetune.md | 34 +++++++++---------- docs/faq/float16.md | 34 +++++++++---------- docs/faq/gradient_compression.md | 34 +++++++++---------- docs/faq/index.md | 34 +++++++++---------- docs/faq/model_parallel_lstm.md | 34 +++++++++---------- docs/faq/multi_devices.md | 34 +++++++++---------- docs/faq/new_op.md | 34 +++++++++---------- docs/faq/nnpack.md | 34 +++++++++---------- docs/faq/perf.md | 34 +++++++++---------- docs/faq/recordio.md | 34 +++++++++---------- docs/faq/s3_integration.md | 34 +++++++++---------- docs/faq/security.md | 34 +++++++++---------- docs/faq/smart_device.md | 34 +++++++++---------- docs/faq/visualize_graph.md | 34 +++++++++---------- docs/faq/why_mxnet.md | 34 +++++++++---------- docs/gluon/index.md | 34 +++++++++---------- docs/index.md | 30 ++++++++-------- docs/install/amazonlinux_setup.md | 30 ++++++++-------- docs/install/build_from_source.md | 34 +++++++++---------- docs/install/c_plus_plus.md | 34 +++++++++---------- docs/install/centos_setup.md | 34 +++++++++---------- docs/install/download.md | 34 +++++++++---------- docs/install/index.md | 34 +++++++++---------- docs/install/java_setup.md | 34 +++++++++---------- docs/install/osx_setup.md | 34 +++++++++---------- docs/install/raspbian_setup.md | 30 ++++++++-------- docs/install/scala_setup.md | 34 +++++++++---------- docs/install/tx2_setup.md | 30 ++++++++-------- docs/install/ubuntu_setup.md | 34 +++++++++---------- docs/install/validate_mxnet.md | 34 +++++++++---------- docs/install/windows_setup.md | 34 +++++++++---------- docs/model_zoo/index.md | 34 +++++++++---------- docs/tutorials/basic/data.md | 34 +++++++++---------- docs/tutorials/basic/index.md | 30 ++++++++-------- docs/tutorials/basic/module.md | 34 +++++++++---------- docs/tutorials/basic/ndarray.md | 34 +++++++++---------- docs/tutorials/basic/ndarray_indexing.md | 34 +++++++++---------- docs/tutorials/basic/reshape_transpose.md | 34 +++++++++---------- docs/tutorials/basic/symbol.md | 34 +++++++++---------- docs/tutorials/c++/basics.md | 34 +++++++++---------- docs/tutorials/c++/index.md | 30 ++++++++-------- .../c++/mxnet_cpp_inference_tutorial.md | 34 +++++++++---------- docs/tutorials/c++/subgraphAPI.md | 34 +++++++++---------- .../control_flow/ControlFlowTutorial.md | 34 +++++++++---------- docs/tutorials/control_flow/index.md | 30 ++++++++-------- docs/tutorials/embedded/index.md | 30 ++++++++-------- docs/tutorials/embedded/wine_detector.md | 34 +++++++++---------- docs/tutorials/gluon/autograd.md | 34 +++++++++---------- docs/tutorials/gluon/custom_layer.md | 34 +++++++++---------- docs/tutorials/gluon/customop.md | 34 +++++++++---------- docs/tutorials/gluon/data_augmentation.md | 34 +++++++++---------- docs/tutorials/gluon/datasets.md | 34 +++++++++---------- docs/tutorials/gluon/gluon.md | 34 +++++++++---------- .../gluon_from_experiment_to_deployment.md | 34 +++++++++---------- .../tutorials/gluon/gotchas_numpy_in_mxnet.md | 34 +++++++++---------- docs/tutorials/gluon/hybrid.md | 34 +++++++++---------- docs/tutorials/gluon/index.md | 30 ++++++++-------- docs/tutorials/gluon/info_gan.md | 34 +++++++++---------- docs/tutorials/gluon/learning_rate_finder.md | 34 +++++++++---------- .../gluon/learning_rate_schedules.md | 34 +++++++++---------- .../gluon/learning_rate_schedules_advanced.md | 34 +++++++++---------- .../gluon/logistic_regression_explained.md | 34 +++++++++---------- docs/tutorials/gluon/mnist.md | 34 +++++++++---------- docs/tutorials/gluon/naming.md | 34 +++++++++---------- docs/tutorials/gluon/ndarray.md | 34 +++++++++---------- docs/tutorials/gluon/pretrained_models.md | 34 +++++++++---------- docs/tutorials/gluon/save_load_params.md | 34 +++++++++---------- docs/tutorials/index.md | 34 +++++++++---------- docs/tutorials/java/index.md | 30 ++++++++-------- docs/tutorials/java/mxnet_java_on_intellij.md | 34 +++++++++---------- docs/tutorials/java/ssd_inference.md | 34 +++++++++---------- docs/tutorials/nlp/cnn.md | 34 +++++++++---------- docs/tutorials/nlp/index.md | 30 ++++++++-------- docs/tutorials/onnx/export_mxnet_to_onnx.md | 34 +++++++++---------- docs/tutorials/onnx/fine_tuning_gluon.md | 34 +++++++++---------- docs/tutorials/onnx/index.md | 30 ++++++++-------- .../tutorials/onnx/inference_on_onnx_model.md | 34 +++++++++---------- docs/tutorials/onnx/super_resolution.md | 34 +++++++++---------- docs/tutorials/python/data_augmentation.md | 34 +++++++++---------- .../python/data_augmentation_with_masks.md | 34 +++++++++---------- docs/tutorials/python/index.md | 30 ++++++++-------- docs/tutorials/python/kvstore.md | 34 +++++++++---------- docs/tutorials/python/linear-regression.md | 34 +++++++++---------- docs/tutorials/python/matrix_factorization.md | 34 +++++++++---------- docs/tutorials/python/mnist.md | 34 +++++++++---------- docs/tutorials/python/predict_image.md | 34 +++++++++---------- docs/tutorials/python/profiler.md | 34 +++++++++---------- .../python/types_of_data_augmentation.md | 34 +++++++++---------- docs/tutorials/r/CallbackFunction.md | 34 +++++++++---------- docs/tutorials/r/CustomIterator.md | 34 +++++++++---------- docs/tutorials/r/CustomLossFunction.md | 34 +++++++++---------- docs/tutorials/r/MultidimLstm.md | 34 +++++++++---------- docs/tutorials/r/charRnnModel.md | 34 +++++++++---------- .../r/classifyRealImageWithPretrainedModel.md | 34 +++++++++---------- docs/tutorials/r/fiveMinutesNeuralNetwork.md | 34 +++++++++---------- docs/tutorials/r/index.md | 34 +++++++++---------- docs/tutorials/r/mnistCompetition.md | 34 +++++++++---------- docs/tutorials/r/ndarray.md | 34 +++++++++---------- docs/tutorials/r/symbol.md | 34 +++++++++---------- docs/tutorials/scala/char_lstm.md | 34 +++++++++---------- docs/tutorials/scala/index.md | 34 +++++++++---------- docs/tutorials/scala/mnist.md | 34 +++++++++---------- .../scala/mxnet_scala_on_intellij.md | 34 +++++++++---------- docs/tutorials/sparse/csr.md | 34 +++++++++---------- docs/tutorials/sparse/index.md | 30 ++++++++-------- docs/tutorials/sparse/row_sparse.md | 34 +++++++++---------- docs/tutorials/sparse/train.md | 34 +++++++++---------- docs/tutorials/speech_recognition/ctc.md | 34 +++++++++---------- docs/tutorials/speech_recognition/index.md | 30 ++++++++-------- docs/tutorials/tensorrt/index.md | 30 ++++++++-------- docs/tutorials/tensorrt/inference_with_trt.md | 34 +++++++++---------- docs/tutorials/unsupervised_learning/gan.md | 34 +++++++++---------- docs/tutorials/unsupervised_learning/index.md | 30 ++++++++-------- docs/tutorials/vision/cnn_visualization.md | 34 +++++++++---------- docs/tutorials/vision/index.md | 30 ++++++++-------- .../vision/large_scale_classification.md | 34 +++++++++---------- example/README.md | 34 +++++++++---------- example/adversary/README.md | 30 ++++++++-------- example/autoencoder/README.md | 34 +++++++++---------- .../variational_autoencoder/README.md | 34 +++++++++---------- example/bayesian-methods/README.md | 34 +++++++++---------- example/bi-lstm-sort/README.md | 34 +++++++++---------- example/caffe/README.md | 34 +++++++++---------- example/capsnet/README.md | 30 ++++++++-------- example/captcha/README.md | 30 ++++++++-------- .../cnn_chinese_text_classification/README.md | 34 +++++++++---------- example/cnn_text_classification/README.md | 34 +++++++++---------- example/ctc/README.md | 34 +++++++++---------- example/deep-embedded-clustering/README.md | 34 +++++++++---------- example/distributed_training/README.md | 34 +++++++++---------- example/dsd/README.md | 34 +++++++++---------- example/fcn-xs/README.md | 34 +++++++++---------- example/gluon/audio/urban_sounds/README.md | 34 +++++++++---------- example/gluon/dc_gan/README.md | 30 ++++++++-------- example/gluon/embedding_learning/README.md | 30 ++++++++-------- example/gluon/lipnet/README.md | 19 +++++++++++ example/gluon/sn_gan/README.md | 34 +++++++++---------- example/gluon/style_transfer/README.md | 34 +++++++++---------- example/gluon/tree_lstm/README.md | 34 +++++++++---------- example/gluon/word_language_model/README.md | 34 +++++++++---------- example/kaggle-ndsb1/README.md | 30 ++++++++-------- example/kaggle-ndsb2/README.md | 34 +++++++++---------- .../matrix_factorization/README.md | 34 +++++++++---------- example/module/README.md | 30 ++++++++-------- example/multi-task/README.md | 34 +++++++++---------- example/multivariate_time_series/README.md | 34 +++++++++---------- example/named_entity_recognition/README.md | 34 +++++++++---------- example/nce-loss/README.md | 34 +++++++++---------- example/neural-style/README.md | 34 +++++++++---------- example/neural-style/end_to_end/README.md | 34 +++++++++---------- example/numpy-ops/README.md | 30 ++++++++-------- example/profiler/README.md | 34 +++++++++---------- example/rcnn/README.md | 30 ++++++++-------- example/recommenders/README.md | 34 +++++++++---------- example/reinforcement-learning/a3c/README.md | 34 +++++++++---------- example/reinforcement-learning/ddpg/README.md | 34 +++++++++---------- .../parallel_actor_critic/README.md | 34 +++++++++---------- .../restricted-boltzmann-machine/README.md | 30 ++++++++-------- example/rnn/README.md | 34 +++++++++---------- example/rnn/bucketing/README.md | 34 +++++++++---------- example/rnn/old/README.md | 34 +++++++++---------- example/rnn/word_lm/README.md | 30 ++++++++-------- .../sparse/factorization_machine/README.md | 34 +++++++++---------- .../sparse/linear_classification/README.md | 34 +++++++++---------- example/sparse/matrix_factorization/README.md | 30 ++++++++-------- example/sparse/wide_deep/README.md | 30 ++++++++-------- example/speech_recognition/README.md | 34 +++++++++---------- example/ssd/README.md | 34 +++++++++---------- example/ssd/dataset/pycocotools/README.md | 30 ++++++++-------- example/ssd/model/README.md | 30 ++++++++-------- example/ssd/symbol/README.md | 30 ++++++++-------- example/ssd/tools/caffe_converter/README.md | 34 +++++++++---------- example/stochastic-depth/README.md | 34 +++++++++---------- example/svm_mnist/README.md | 34 +++++++++---------- example/svrg_module/README.md | 34 +++++++++---------- example/vae-gan/README.md | 34 +++++++++---------- julia/LICENSE.md | 30 ++++++++-------- julia/NEWS.md | 34 +++++++++---------- julia/README-DEV.md | 30 ++++++++-------- julia/README.md | 34 +++++++++---------- julia/docs/src/api.md | 30 ++++++++-------- julia/docs/src/api/callback.md | 30 ++++++++-------- julia/docs/src/api/context.md | 30 ++++++++-------- julia/docs/src/api/executor.md | 30 ++++++++-------- julia/docs/src/api/initializer.md | 30 ++++++++-------- julia/docs/src/api/io.md | 34 +++++++++---------- julia/docs/src/api/kvstore.md | 30 ++++++++-------- julia/docs/src/api/metric.md | 30 ++++++++-------- julia/docs/src/api/model.md | 30 ++++++++-------- julia/docs/src/api/ndarray.md | 34 +++++++++---------- julia/docs/src/api/nn-factory.md | 30 ++++++++-------- julia/docs/src/api/symbolic-node.md | 30 ++++++++-------- julia/docs/src/api/visualize.md | 30 ++++++++-------- julia/docs/src/index.md | 30 ++++++++-------- julia/docs/src/tutorial/char-lstm.md | 34 +++++++++---------- julia/docs/src/tutorial/mnist.md | 34 +++++++++---------- julia/docs/src/user-guide/faq.md | 30 ++++++++-------- julia/docs/src/user-guide/install.md | 34 +++++++++---------- julia/docs/src/user-guide/overview.md | 34 +++++++++---------- julia/examples/char-lstm/README.md | 34 +++++++++---------- julia/plugins/README.md | 34 +++++++++---------- matlab/README.md | 34 +++++++++---------- .../examples/gluon/style_transfer/README.md | 34 +++++++++---------- .../sparse/matrix_factorization/README.md | 30 ++++++++-------- .../examples/sparse/wide_deep/README.md | 30 ++++++++-------- perl-package/README.md | 30 ++++++++-------- plugin/caffe/README.md | 34 +++++++++---------- python/README.md | 34 +++++++++---------- python/minpy/README.md | 21 ++++++++++++ scala-package/README.md | 34 +++++++++---------- .../javaapi/infer/objectdetector/README.md | 34 +++++++++---------- .../javaapi/infer/predictor/README.md | 34 +++++++++---------- .../apache/mxnetexamples/benchmark/README.md | 30 ++++++++-------- .../cnntextclassification/README.md | 30 ++++++++-------- .../apache/mxnetexamples/customop/README.md | 30 ++++++++-------- .../org/apache/mxnetexamples/gan/README.md | 30 ++++++++-------- .../mxnetexamples/imclassification/README.md | 34 +++++++++---------- .../infer/imageclassifier/README.md | 34 +++++++++---------- .../infer/objectdetector/README.md | 34 +++++++++---------- .../mxnetexamples/neuralstyle/README.md | 34 +++++++++---------- .../org/apache/mxnetexamples/rnn/README.md | 30 ++++++++-------- scala-package/memory-management.md | 34 +++++++++---------- scala-package/mxnet-demo/java-demo/README.md | 34 +++++++++---------- scala-package/mxnet-demo/scala-demo/README.md | 30 ++++++++-------- scala-package/native/README.md | 34 +++++++++---------- scala-package/packageTest/README.md | 34 +++++++++---------- scala-package/spark/README.md | 34 +++++++++---------- tests/README.md | 30 ++++++++-------- tests/nightly/README.md | 34 +++++++++---------- .../apache_rat_license_check/README.md | 34 +++++++++---------- .../broken_link_checker_test/README.md | 30 ++++++++-------- .../README.md | 30 ++++++++-------- tests/nightly/straight_dope/README.md | 30 ++++++++-------- tests/python/README.md | 30 ++++++++-------- tools/accnn/README.md | 30 ++++++++-------- tools/bandwidth/README.md | 34 +++++++++---------- tools/caffe_converter/README.md | 30 ++++++++-------- tools/caffe_translator/README.md | 34 +++++++++---------- tools/caffe_translator/build_from_source.md | 30 ++++++++-------- tools/caffe_translator/faq.md | 34 +++++++++---------- tools/cfn/Readme.md | 30 ++++++++-------- tools/coreml/README.md | 34 +++++++++---------- tools/dependencies/README.md | 34 +++++++++---------- tools/staticbuild/README.md | 34 +++++++++---------- 361 files changed, 5620 insertions(+), 6298 deletions(-) create mode 100644 python/minpy/README.md diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 96736b76f136..ffe0da657aae 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + Contributors of Apache MXNet (incubating) ========================================= diff --git a/MKLDNN_README.md b/MKLDNN_README.md index 7068b8519b39..214fc83985fb 100644 --- a/MKLDNN_README.md +++ b/MKLDNN_README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Build/Install MXNet with MKL-DNN diff --git a/NEWS.md b/NEWS.md index 95fa8a52b047..1af4b138e600 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + MXNet Change Log ================ diff --git a/README.md b/README.md index 6955ddd3ed3d..6a8ecdd99bd0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + +

diff --git a/amalgamation/README.md b/amalgamation/README.md index aaf01b6b6e7f..2ecf1626c1e7 100644 --- a/amalgamation/README.md +++ b/amalgamation/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + MXNet Amalgamation ================== diff --git a/ci/README.md b/ci/README.md index 6ad8cffa1550..155a0104a125 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Containerized build & test utilities diff --git a/ci/docker/qemu/README.md b/ci/docker/qemu/README.md index bf63ed9fc89c..c06b34562b57 100644 --- a/ci/docker/qemu/README.md +++ b/ci/docker/qemu/README.md @@ -1,20 +1,18 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + These are files used in the docker container that runs QEMU diff --git a/ci/publish/README.md b/ci/publish/README.md index 2beb2b399f8b..cdd70ce82258 100644 --- a/ci/publish/README.md +++ b/ci/publish/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Publish Settings diff --git a/ci/qemu/README.md b/ci/qemu/README.md index 4b5e0db3047f..498f8b7a8739 100644 --- a/ci/qemu/README.md +++ b/ci/qemu/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # QEMU base image creation diff --git a/contrib/clojure-package/README.md b/contrib/clojure-package/README.md index 6c20fe0b609c..7566ade66ce8 100644 --- a/contrib/clojure-package/README.md +++ b/contrib/clojure-package/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Clojure MXNet diff --git a/contrib/clojure-package/examples/captcha/README.md b/contrib/clojure-package/examples/captcha/README.md index 51a407b6e04c..be71e07b2120 100644 --- a/contrib/clojure-package/examples/captcha/README.md +++ b/contrib/clojure-package/examples/captcha/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Captcha diff --git a/contrib/clojure-package/examples/cnn-text-classification/README.md b/contrib/clojure-package/examples/cnn-text-classification/README.md index b1ae538e5281..f2ed939bee16 100644 --- a/contrib/clojure-package/examples/cnn-text-classification/README.md +++ b/contrib/clojure-package/examples/cnn-text-classification/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # cnn-text-classification diff --git a/contrib/clojure-package/examples/gan/README.md b/contrib/clojure-package/examples/gan/README.md index 8c2c195f0ccc..9c4bea8ed586 100644 --- a/contrib/clojure-package/examples/gan/README.md +++ b/contrib/clojure-package/examples/gan/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # gan diff --git a/contrib/clojure-package/examples/imclassification/README.md b/contrib/clojure-package/examples/imclassification/README.md index f9aa30f1b0c9..06ae9461816e 100644 --- a/contrib/clojure-package/examples/imclassification/README.md +++ b/contrib/clojure-package/examples/imclassification/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # imclassification diff --git a/contrib/clojure-package/examples/infer/imageclassifier/README.md b/contrib/clojure-package/examples/infer/imageclassifier/README.md index ec772d2c3980..ef82feebd53f 100644 --- a/contrib/clojure-package/examples/infer/imageclassifier/README.md +++ b/contrib/clojure-package/examples/infer/imageclassifier/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # imageclassifier diff --git a/contrib/clojure-package/examples/infer/objectdetector/README.md b/contrib/clojure-package/examples/infer/objectdetector/README.md index b9e216eb17ad..3a38e7d3907e 100644 --- a/contrib/clojure-package/examples/infer/objectdetector/README.md +++ b/contrib/clojure-package/examples/infer/objectdetector/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # objectdetector diff --git a/contrib/clojure-package/examples/infer/predictor/README.md b/contrib/clojure-package/examples/infer/predictor/README.md index 9b9be7f7a35a..9a844c4b9353 100644 --- a/contrib/clojure-package/examples/infer/predictor/README.md +++ b/contrib/clojure-package/examples/infer/predictor/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # predictor diff --git a/contrib/clojure-package/examples/module/README.md b/contrib/clojure-package/examples/module/README.md index 421720fe3896..fcd16b74a173 100644 --- a/contrib/clojure-package/examples/module/README.md +++ b/contrib/clojure-package/examples/module/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + ## Instructions diff --git a/contrib/clojure-package/examples/multi-label/README.md b/contrib/clojure-package/examples/multi-label/README.md index bc2a0cca3e88..aebb83be985d 100644 --- a/contrib/clojure-package/examples/multi-label/README.md +++ b/contrib/clojure-package/examples/multi-label/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # multi-label diff --git a/contrib/clojure-package/examples/neural-style/README.md b/contrib/clojure-package/examples/neural-style/README.md index fa27e49eab7a..209e56ae20d2 100644 --- a/contrib/clojure-package/examples/neural-style/README.md +++ b/contrib/clojure-package/examples/neural-style/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # neural-style diff --git a/contrib/clojure-package/examples/pre-trained-models/README.md b/contrib/clojure-package/examples/pre-trained-models/README.md index b3df9af8ef8d..e1aefbca0c5b 100644 --- a/contrib/clojure-package/examples/pre-trained-models/README.md +++ b/contrib/clojure-package/examples/pre-trained-models/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # pre-trained-models diff --git a/contrib/clojure-package/examples/profiler/README.md b/contrib/clojure-package/examples/profiler/README.md index 73482c09d6ee..8ab37d713c09 100644 --- a/contrib/clojure-package/examples/profiler/README.md +++ b/contrib/clojure-package/examples/profiler/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # profiler diff --git a/contrib/clojure-package/examples/rnn/README.md b/contrib/clojure-package/examples/rnn/README.md index 379e1e40c65a..640b63d43f10 100644 --- a/contrib/clojure-package/examples/rnn/README.md +++ b/contrib/clojure-package/examples/rnn/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # rnn diff --git a/contrib/clojure-package/examples/tutorial/README.md b/contrib/clojure-package/examples/tutorial/README.md index 625002e06bdd..54f05b36f800 100644 --- a/contrib/clojure-package/examples/tutorial/README.md +++ b/contrib/clojure-package/examples/tutorial/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # tutorial diff --git a/contrib/clojure-package/examples/visualization/README.md b/contrib/clojure-package/examples/visualization/README.md index 6e951542ddca..0638d9c49c5d 100644 --- a/contrib/clojure-package/examples/visualization/README.md +++ b/contrib/clojure-package/examples/visualization/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # visualization diff --git a/contrib/clojure-package/testing.md b/contrib/clojure-package/testing.md index 66de164da540..0e87790686f0 100644 --- a/contrib/clojure-package/testing.md +++ b/contrib/clojure-package/testing.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + ## Help with Testing diff --git a/cpp-package/README.md b/cpp-package/README.md index 524fbadb87ee..165a65eaac6a 100644 --- a/cpp-package/README.md +++ b/cpp-package/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet C++ Package diff --git a/cpp-package/example/README.md b/cpp-package/example/README.md index 00c6e96f6f27..555316dd1ac3 100644 --- a/cpp-package/example/README.md +++ b/cpp-package/example/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet C++ Package Examples diff --git a/cpp-package/example/feature_extract/README.md b/cpp-package/example/feature_extract/README.md index 7241bcd6425a..0b94bef7705f 100644 --- a/cpp-package/example/feature_extract/README.md +++ b/cpp-package/example/feature_extract/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + This example shows how to extract features with a pretrained model. diff --git a/cpp-package/example/inference/README.md b/cpp-package/example/inference/README.md index 322c92c75208..865388b549db 100644 --- a/cpp-package/example/inference/README.md +++ b/cpp-package/example/inference/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet C++ Package Inference Workflow Examples diff --git a/docs/README.md b/docs/README.md index 6ff2ae5efe18..c0b40362b7e2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Building and Updating MXNet Documentation diff --git a/docs/api/c++/index.md b/docs/api/c++/index.md index 491e91ab5e5d..e09c16e60b28 100644 --- a/docs/api/c++/index.md +++ b/docs/api/c++/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet - C++ API diff --git a/docs/api/clojure/index.md b/docs/api/clojure/index.md index cb27c637d586..38e1c5ff9d1e 100644 --- a/docs/api/clojure/index.md +++ b/docs/api/clojure/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet - Clojure API diff --git a/docs/api/clojure/kvstore.md b/docs/api/clojure/kvstore.md index d5f7c089ff7c..36af3a8e6349 100644 --- a/docs/api/clojure/kvstore.md +++ b/docs/api/clojure/kvstore.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # KVStore API diff --git a/docs/api/clojure/module.md b/docs/api/clojure/module.md index db7cb70d0fce..e1414adeb674 100644 --- a/docs/api/clojure/module.md +++ b/docs/api/clojure/module.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Module API The module API provides an intermediate and high-level interface for performing computation with neural networks in MXNet. Module wraps a Symbol and one or more Executors. It has both a high level and intermediate level API. diff --git a/docs/api/clojure/ndarray.md b/docs/api/clojure/ndarray.md index 3d367c6c5011..991e458ccc04 100644 --- a/docs/api/clojure/ndarray.md +++ b/docs/api/clojure/ndarray.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # NDArray API diff --git a/docs/api/clojure/symbol.md b/docs/api/clojure/symbol.md index a70c9ea5b906..2457dba6d530 100644 --- a/docs/api/clojure/symbol.md +++ b/docs/api/clojure/symbol.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Clojure Symbolic API diff --git a/docs/api/clojure/symbol_in_pictures.md b/docs/api/clojure/symbol_in_pictures.md index aac133d2d067..f5a56e8dcc3b 100644 --- a/docs/api/clojure/symbol_in_pictures.md +++ b/docs/api/clojure/symbol_in_pictures.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Symbolic Configuration and Execution in Pictures diff --git a/docs/api/index.md b/docs/api/index.md index 62de6cbed502..a1164fc72477 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # MXNet APIs diff --git a/docs/api/java/index.md b/docs/api/java/index.md index 9955c05f3bc1..099848cdc01b 100644 --- a/docs/api/java/index.md +++ b/docs/api/java/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet - Java Inference API diff --git a/docs/api/julia/index.md b/docs/api/julia/index.md index 0c5c93c9c4db..8aa884eb2772 100644 --- a/docs/api/julia/index.md +++ b/docs/api/julia/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet - Julia API diff --git a/docs/api/perl/index.md b/docs/api/perl/index.md index 0aef4d4aa4d3..2d3d2656d7cd 100644 --- a/docs/api/perl/index.md +++ b/docs/api/perl/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet - Perl API diff --git a/docs/api/perl/io.md b/docs/api/perl/io.md index c3f55c6e27ea..be4976425553 100644 --- a/docs/api/perl/io.md +++ b/docs/api/perl/io.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Data Loading API diff --git a/docs/api/perl/kvstore.md b/docs/api/perl/kvstore.md index e221dbbf4f9b..4556f68bd837 100644 --- a/docs/api/perl/kvstore.md +++ b/docs/api/perl/kvstore.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # KVStore API diff --git a/docs/api/perl/module.md b/docs/api/perl/module.md index 314b5e8d41da..7b7054646d62 100644 --- a/docs/api/perl/module.md +++ b/docs/api/perl/module.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Module API diff --git a/docs/api/perl/ndarray.md b/docs/api/perl/ndarray.md index 4e7e3a3562ce..2ffef392261d 100644 --- a/docs/api/perl/ndarray.md +++ b/docs/api/perl/ndarray.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # NDArray API diff --git a/docs/api/perl/symbol.md b/docs/api/perl/symbol.md index d9d7d7143c26..e3ad4816daa4 100644 --- a/docs/api/perl/symbol.md +++ b/docs/api/perl/symbol.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Perl Symbolic API diff --git a/docs/api/python/autograd/autograd.md b/docs/api/python/autograd/autograd.md index 38a99f504f60..1905831b16d9 100644 --- a/docs/api/python/autograd/autograd.md +++ b/docs/api/python/autograd/autograd.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Autograd Package diff --git a/docs/api/python/callback/callback.md b/docs/api/python/callback/callback.md index be99dc01ee7b..225ffcb9a2fd 100644 --- a/docs/api/python/callback/callback.md +++ b/docs/api/python/callback/callback.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Callback API diff --git a/docs/api/python/contrib/contrib.md b/docs/api/python/contrib/contrib.md index 65922556d2cc..4fe80625766c 100644 --- a/docs/api/python/contrib/contrib.md +++ b/docs/api/python/contrib/contrib.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Contrib Package diff --git a/docs/api/python/contrib/onnx.md b/docs/api/python/contrib/onnx.md index 18b108caa5b5..5691fb022959 100644 --- a/docs/api/python/contrib/onnx.md +++ b/docs/api/python/contrib/onnx.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # ONNX-MXNet API diff --git a/docs/api/python/contrib/svrg_optimization.md b/docs/api/python/contrib/svrg_optimization.md index c79dd44f003b..9c5c3a02c88f 100644 --- a/docs/api/python/contrib/svrg_optimization.md +++ b/docs/api/python/contrib/svrg_optimization.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # SVRG Optimization in Python Module API diff --git a/docs/api/python/contrib/text.md b/docs/api/python/contrib/text.md index 1fd9514e0f65..577865f501cb 100644 --- a/docs/api/python/contrib/text.md +++ b/docs/api/python/contrib/text.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Text API diff --git a/docs/api/python/executor/executor.md b/docs/api/python/executor/executor.md index 26c97907dfc5..2cf5062e1ed3 100644 --- a/docs/api/python/executor/executor.md +++ b/docs/api/python/executor/executor.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Executor and Executor Manager diff --git a/docs/api/python/gluon/contrib.md b/docs/api/python/gluon/contrib.md index f6e6dfe771a5..b893d5841254 100644 --- a/docs/api/python/gluon/contrib.md +++ b/docs/api/python/gluon/contrib.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Gluon Contrib API diff --git a/docs/api/python/gluon/data.md b/docs/api/python/gluon/data.md index 6d4440f91709..b695e1bc3209 100644 --- a/docs/api/python/gluon/data.md +++ b/docs/api/python/gluon/data.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Gluon Data API diff --git a/docs/api/python/gluon/gluon.md b/docs/api/python/gluon/gluon.md index cbb0750e47a2..c063a71d43ea 100644 --- a/docs/api/python/gluon/gluon.md +++ b/docs/api/python/gluon/gluon.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Gluon Package diff --git a/docs/api/python/gluon/loss.md b/docs/api/python/gluon/loss.md index a2ac65f84b53..8e8f92c5ee49 100644 --- a/docs/api/python/gluon/loss.md +++ b/docs/api/python/gluon/loss.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Gluon Loss API diff --git a/docs/api/python/gluon/model_zoo.md b/docs/api/python/gluon/model_zoo.md index 07d11cd7242e..4672a3b7ad81 100644 --- a/docs/api/python/gluon/model_zoo.md +++ b/docs/api/python/gluon/model_zoo.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Gluon Model Zoo diff --git a/docs/api/python/gluon/nn.md b/docs/api/python/gluon/nn.md index 544a315eb784..2ec66f10416b 100644 --- a/docs/api/python/gluon/nn.md +++ b/docs/api/python/gluon/nn.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Gluon Neural Network Layers diff --git a/docs/api/python/gluon/rnn.md b/docs/api/python/gluon/rnn.md index e7da65741f19..f639479d5db3 100644 --- a/docs/api/python/gluon/rnn.md +++ b/docs/api/python/gluon/rnn.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Gluon Recurrent Neural Network API diff --git a/docs/api/python/image/image.md b/docs/api/python/image/image.md index 61cca225a230..91e4a6b3ec1a 100644 --- a/docs/api/python/image/image.md +++ b/docs/api/python/image/image.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Image API diff --git a/docs/api/python/index.md b/docs/api/python/index.md index 1d11eb7182a1..6130078ee796 100644 --- a/docs/api/python/index.md +++ b/docs/api/python/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet - Python API diff --git a/docs/api/python/io/io.md b/docs/api/python/io/io.md index 057d64c129a4..c0dc8d1efb1d 100644 --- a/docs/api/python/io/io.md +++ b/docs/api/python/io/io.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Data Loading API diff --git a/docs/api/python/kvstore/kvstore.md b/docs/api/python/kvstore/kvstore.md index 3a2957353620..7858ccd02699 100644 --- a/docs/api/python/kvstore/kvstore.md +++ b/docs/api/python/kvstore/kvstore.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # KVStore API diff --git a/docs/api/python/metric/metric.md b/docs/api/python/metric/metric.md index 5423971ad6fb..a3e3b545a56d 100644 --- a/docs/api/python/metric/metric.md +++ b/docs/api/python/metric/metric.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Evaluation Metric API diff --git a/docs/api/python/model.md b/docs/api/python/model.md index a98d2e4119db..967765740aff 100644 --- a/docs/api/python/model.md +++ b/docs/api/python/model.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Model API diff --git a/docs/api/python/module/module.md b/docs/api/python/module/module.md index 51c2343deb58..86ffad559897 100644 --- a/docs/api/python/module/module.md +++ b/docs/api/python/module/module.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Module API diff --git a/docs/api/python/ndarray/contrib.md b/docs/api/python/ndarray/contrib.md index 8d37ca41f4ba..f60e7f141adf 100644 --- a/docs/api/python/ndarray/contrib.md +++ b/docs/api/python/ndarray/contrib.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Contrib NDArray API diff --git a/docs/api/python/ndarray/linalg.md b/docs/api/python/ndarray/linalg.md index fac44c0f06ef..41436c3ba2d1 100644 --- a/docs/api/python/ndarray/linalg.md +++ b/docs/api/python/ndarray/linalg.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Linear Algebra NDArray API diff --git a/docs/api/python/ndarray/ndarray.md b/docs/api/python/ndarray/ndarray.md index b71b961fef3c..3cebffccaaee 100644 --- a/docs/api/python/ndarray/ndarray.md +++ b/docs/api/python/ndarray/ndarray.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # NDArray API diff --git a/docs/api/python/ndarray/random.md b/docs/api/python/ndarray/random.md index fc8c25900415..74e7d082df27 100644 --- a/docs/api/python/ndarray/random.md +++ b/docs/api/python/ndarray/random.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Random Distribution Generator NDArray API diff --git a/docs/api/python/ndarray/sparse.md b/docs/api/python/ndarray/sparse.md index 0a513dc7a624..71dfaafbf294 100644 --- a/docs/api/python/ndarray/sparse.md +++ b/docs/api/python/ndarray/sparse.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Sparse NDArray API diff --git a/docs/api/python/optimization/contrib.md b/docs/api/python/optimization/contrib.md index d67887992e50..a431508aa22d 100644 --- a/docs/api/python/optimization/contrib.md +++ b/docs/api/python/optimization/contrib.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Contrib Optimization API diff --git a/docs/api/python/optimization/optimization.md b/docs/api/python/optimization/optimization.md index 56bf8f538ed8..03448123a14f 100644 --- a/docs/api/python/optimization/optimization.md +++ b/docs/api/python/optimization/optimization.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Optimization: initialize and update weights diff --git a/docs/api/python/rtc/rtc.md b/docs/api/python/rtc/rtc.md index 5f90f3a8b77c..8f5f4e773efc 100644 --- a/docs/api/python/rtc/rtc.md +++ b/docs/api/python/rtc/rtc.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Run-Time Compilation API diff --git a/docs/api/python/symbol/contrib.md b/docs/api/python/symbol/contrib.md index 2429b3765ace..2a6a5efe29be 100644 --- a/docs/api/python/symbol/contrib.md +++ b/docs/api/python/symbol/contrib.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Contrib Symbol API diff --git a/docs/api/python/symbol/linalg.md b/docs/api/python/symbol/linalg.md index c04846e7ba92..f1891e29f896 100644 --- a/docs/api/python/symbol/linalg.md +++ b/docs/api/python/symbol/linalg.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Linear Algebra Symbol API diff --git a/docs/api/python/symbol/random.md b/docs/api/python/symbol/random.md index 1c5f916ad10a..b62747c6ee7b 100644 --- a/docs/api/python/symbol/random.md +++ b/docs/api/python/symbol/random.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Random Distribution Generator Symbol API diff --git a/docs/api/python/symbol/rnn.md b/docs/api/python/symbol/rnn.md index 627d2e0cde37..287e5a178ad2 100644 --- a/docs/api/python/symbol/rnn.md +++ b/docs/api/python/symbol/rnn.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # RNN Cell Symbol API diff --git a/docs/api/python/symbol/sparse.md b/docs/api/python/symbol/sparse.md index 866a302940cc..473bb4bbe530 100644 --- a/docs/api/python/symbol/sparse.md +++ b/docs/api/python/symbol/sparse.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Sparse Symbol API diff --git a/docs/api/python/symbol/symbol.md b/docs/api/python/symbol/symbol.md index 9783053d9a1d..9cab2c59e862 100644 --- a/docs/api/python/symbol/symbol.md +++ b/docs/api/python/symbol/symbol.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Symbol API diff --git a/docs/api/python/symbol_in_pictures/symbol_in_pictures.md b/docs/api/python/symbol_in_pictures/symbol_in_pictures.md index 5326ade3003f..23a9757b83d6 100644 --- a/docs/api/python/symbol_in_pictures/symbol_in_pictures.md +++ b/docs/api/python/symbol_in_pictures/symbol_in_pictures.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Symbolic Configuration and Execution in Pictures diff --git a/docs/api/python/tools/test_utils.md b/docs/api/python/tools/test_utils.md index 2896a594c004..a442e9b46941 100644 --- a/docs/api/python/tools/test_utils.md +++ b/docs/api/python/tools/test_utils.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Test Utilities diff --git a/docs/api/python/tools/visualization.md b/docs/api/python/tools/visualization.md index 8ba722878616..c807ce6fbe05 100644 --- a/docs/api/python/tools/visualization.md +++ b/docs/api/python/tools/visualization.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Visualization diff --git a/docs/api/r/index.md b/docs/api/r/index.md index 5a85a770d80e..e8032cf62bfb 100644 --- a/docs/api/r/index.md +++ b/docs/api/r/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet - R API diff --git a/docs/api/scala/index.md b/docs/api/scala/index.md index 76e5673aa3bf..bda1945b769d 100644 --- a/docs/api/scala/index.md +++ b/docs/api/scala/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet - Scala API diff --git a/docs/api/scala/infer.md b/docs/api/scala/infer.md index af50fc2e4b03..d9a4a62bb7c7 100644 --- a/docs/api/scala/infer.md +++ b/docs/api/scala/infer.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Infer API The MXNet Scala Infer API provides you with model loading and inference functionality using the MXNet Scala package. diff --git a/docs/api/scala/io.md b/docs/api/scala/io.md index fe819e876bbd..9a7a865f9ea5 100644 --- a/docs/api/scala/io.md +++ b/docs/api/scala/io.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Scala Data Loading API This topic introduces the data input method for MXNet. MXNet uses an iterator to provide data to the neural network. Iterators do some preprocessing and generate batches for the neural network. diff --git a/docs/api/scala/kvstore.md b/docs/api/scala/kvstore.md index 32e3c1f1145c..30ea97d207f6 100644 --- a/docs/api/scala/kvstore.md +++ b/docs/api/scala/kvstore.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # KVStore API diff --git a/docs/api/scala/model.md b/docs/api/scala/model.md index 4eeae42de6be..12156132d35a 100644 --- a/docs/api/scala/model.md +++ b/docs/api/scala/model.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Scala Model API diff --git a/docs/api/scala/module.md b/docs/api/scala/module.md index c7f9a0a29a3b..09e7c657c12e 100644 --- a/docs/api/scala/module.md +++ b/docs/api/scala/module.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Module API The module API provides an intermediate and high-level interface for performing computation with neural networks in MXNet. A *module* is an instance of subclasses of the `BaseModule`. The most widely used module class is called `Module`. Module wraps a `Symbol` and one or more `Executors`. For a full list of functions, see `BaseModule`. diff --git a/docs/api/scala/ndarray.md b/docs/api/scala/ndarray.md index ad3349d02c8e..0c13669d64ab 100644 --- a/docs/api/scala/ndarray.md +++ b/docs/api/scala/ndarray.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # NDArray API diff --git a/docs/api/scala/symbol.md b/docs/api/scala/symbol.md index 52d63e91f38b..aaddc2a8a2f0 100644 --- a/docs/api/scala/symbol.md +++ b/docs/api/scala/symbol.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Scala Symbolic API diff --git a/docs/api/scala/symbol_in_pictures.md b/docs/api/scala/symbol_in_pictures.md index 9d6ecdd55202..23ece95e835a 100644 --- a/docs/api/scala/symbol_in_pictures.md +++ b/docs/api/scala/symbol_in_pictures.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Symbolic Configuration and Execution in Pictures diff --git a/docs/architecture/exception_handling.md b/docs/architecture/exception_handling.md index 08b05058628e..6a9ab9ae0c4c 100644 --- a/docs/architecture/exception_handling.md +++ b/docs/architecture/exception_handling.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Exception Handling in MXNet diff --git a/docs/architecture/index.md b/docs/architecture/index.md index d17564090c89..3c71edfeeb05 100644 --- a/docs/architecture/index.md +++ b/docs/architecture/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Architecture diff --git a/docs/architecture/note_data_loading.md b/docs/architecture/note_data_loading.md index 83420a3ff071..293b67572cb3 100644 --- a/docs/architecture/note_data_loading.md +++ b/docs/architecture/note_data_loading.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Designing Efficient Data Loaders for Deep Learning diff --git a/docs/architecture/note_engine.md b/docs/architecture/note_engine.md index 378191cdb184..6d0e1e19fcef 100644 --- a/docs/architecture/note_engine.md +++ b/docs/architecture/note_engine.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Dependency Engine for Deep Learning diff --git a/docs/architecture/note_memory.md b/docs/architecture/note_memory.md index cd161c9ed0f0..e8b795bbd5c4 100644 --- a/docs/architecture/note_memory.md +++ b/docs/architecture/note_memory.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Optimizing Memory Consumption in Deep Learning diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 04abacd869cd..fefa1e8a0a0c 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet System Architecture diff --git a/docs/architecture/program_model.md b/docs/architecture/program_model.md index b4d2c31accda..2731a3df7394 100644 --- a/docs/architecture/program_model.md +++ b/docs/architecture/program_model.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Deep Learning Programming Style diff --git a/docs/architecture/rnn_interface.md b/docs/architecture/rnn_interface.md index 523295e934df..5cd5a7b85835 100644 --- a/docs/architecture/rnn_interface.md +++ b/docs/architecture/rnn_interface.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Survey of Existing Interfaces and Implementations diff --git a/docs/build_version_doc/README.md b/docs/build_version_doc/README.md index 2e30a3bbe5f9..4821f50f5d6e 100644 --- a/docs/build_version_doc/README.md +++ b/docs/build_version_doc/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Documentation Build Scripts diff --git a/docs/community/contribute.md b/docs/community/contribute.md index 727b9d137784..50aa1a607a44 100644 --- a/docs/community/contribute.md +++ b/docs/community/contribute.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Contributing to MXNet diff --git a/docs/community/ecosystem.md b/docs/community/ecosystem.md index 792f9de6ff3c..101f76d4c380 100644 --- a/docs/community/ecosystem.md +++ b/docs/community/ecosystem.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet's Ecosystem diff --git a/docs/community/index.md b/docs/community/index.md index a2be9861db96..5db4253a65df 100644 --- a/docs/community/index.md +++ b/docs/community/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # MXNet Community diff --git a/docs/community/mxnet_channels.md b/docs/community/mxnet_channels.md index ec0149769474..df05855dfedd 100644 --- a/docs/community/mxnet_channels.md +++ b/docs/community/mxnet_channels.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Join MXNet Development Discussion diff --git a/docs/community/powered_by.md b/docs/community/powered_by.md index 0cca3d02f0c1..4b854c4afdc6 100644 --- a/docs/community/powered_by.md +++ b/docs/community/powered_by.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Powered By diff --git a/docs/faq/add_op_in_backend.md b/docs/faq/add_op_in_backend.md index 64f6c8ae0f98..0e734d62bce2 100644 --- a/docs/faq/add_op_in_backend.md +++ b/docs/faq/add_op_in_backend.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # A Beginner's Guide to Implementing Operators in MXNet Backend diff --git a/docs/faq/bucketing.md b/docs/faq/bucketing.md index 49994dc4fe97..b5fb987d23e4 100644 --- a/docs/faq/bucketing.md +++ b/docs/faq/bucketing.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Bucketing in MXNet When we train recurrent neural networks (RNNs), we _unroll_ the network in time. diff --git a/docs/faq/caffe.md b/docs/faq/caffe.md index b99acb07e0bd..e0362ca2776b 100644 --- a/docs/faq/caffe.md +++ b/docs/faq/caffe.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # How to | Convert from Caffe to MXNet diff --git a/docs/faq/cloud.md b/docs/faq/cloud.md index bd2987f0a057..ecc0f46e02c0 100644 --- a/docs/faq/cloud.md +++ b/docs/faq/cloud.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # MXNet on the Cloud diff --git a/docs/faq/develop_and_hack.md b/docs/faq/develop_and_hack.md index b09824dc2ef6..0e7d221f7dc3 100644 --- a/docs/faq/develop_and_hack.md +++ b/docs/faq/develop_and_hack.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Develop and Hack MXNet - [Create new operators](new_op.md) diff --git a/docs/faq/distributed_training.md b/docs/faq/distributed_training.md index 32696f34e250..4d4a220c5661 100644 --- a/docs/faq/distributed_training.md +++ b/docs/faq/distributed_training.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Distributed Training in MXNet MXNet supports distributed training enabling us to leverage multiple machines for faster training. diff --git a/docs/faq/env_var.md b/docs/faq/env_var.md index 2f649bedafc1..c35d4e5723a5 100644 --- a/docs/faq/env_var.md +++ b/docs/faq/env_var.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Environment Variables ===================== diff --git a/docs/faq/faq.md b/docs/faq/faq.md index 4682c95d2a47..b7866be5c03a 100644 --- a/docs/faq/faq.md +++ b/docs/faq/faq.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Frequently Asked Questions diff --git a/docs/faq/finetune.md b/docs/faq/finetune.md index 33a4ffb32521..4a03eefa05c9 100644 --- a/docs/faq/finetune.md +++ b/docs/faq/finetune.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Fine-tune with Pretrained Models diff --git a/docs/faq/float16.md b/docs/faq/float16.md index 0fda178c4bde..323218ce7df6 100644 --- a/docs/faq/float16.md +++ b/docs/faq/float16.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Mixed precision training using float16 diff --git a/docs/faq/gradient_compression.md b/docs/faq/gradient_compression.md index adf7f62bc193..9e926c52feca 100644 --- a/docs/faq/gradient_compression.md +++ b/docs/faq/gradient_compression.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Gradient Compression diff --git a/docs/faq/index.md b/docs/faq/index.md index 99c448da027d..92f1ccde00f4 100644 --- a/docs/faq/index.md +++ b/docs/faq/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet FAQ diff --git a/docs/faq/model_parallel_lstm.md b/docs/faq/model_parallel_lstm.md index 63f4db6d9df9..e9a8768b6fec 100644 --- a/docs/faq/model_parallel_lstm.md +++ b/docs/faq/model_parallel_lstm.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Training with Multiple GPUs Using Model Parallelism Training deep learning models can be resource intensive. diff --git a/docs/faq/multi_devices.md b/docs/faq/multi_devices.md index 46ec837025d9..60a8003f75d0 100644 --- a/docs/faq/multi_devices.md +++ b/docs/faq/multi_devices.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Run MXNet on Multiple CPU/GPUs with Data Parallelism diff --git a/docs/faq/new_op.md b/docs/faq/new_op.md index 910bb64ef5d9..4d51eaf8059d 100644 --- a/docs/faq/new_op.md +++ b/docs/faq/new_op.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # How to Create New Operators (Layers) diff --git a/docs/faq/nnpack.md b/docs/faq/nnpack.md index 690315efe54a..c0b27b9bf4e6 100644 --- a/docs/faq/nnpack.md +++ b/docs/faq/nnpack.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + ### NNPACK for Multi-Core CPU Support in MXNet [NNPACK](https://github.com/Maratyszcza/NNPACK) is an acceleration package diff --git a/docs/faq/perf.md b/docs/faq/perf.md index d00c904c0873..00310dfbb5bd 100644 --- a/docs/faq/perf.md +++ b/docs/faq/perf.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Some Tips for Improving MXNet Performance Even after fixing the training or deployment environment and parallelization scheme, diff --git a/docs/faq/recordio.md b/docs/faq/recordio.md index 3ece38617513..7367b524224d 100644 --- a/docs/faq/recordio.md +++ b/docs/faq/recordio.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + ## Create a Dataset Using RecordIO diff --git a/docs/faq/s3_integration.md b/docs/faq/s3_integration.md index 18bd38df71df..49520b5814e1 100644 --- a/docs/faq/s3_integration.md +++ b/docs/faq/s3_integration.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Use data from S3 for training diff --git a/docs/faq/security.md b/docs/faq/security.md index 05153e20aaf1..88d48be35ecb 100644 --- a/docs/faq/security.md +++ b/docs/faq/security.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Security Best Practices diff --git a/docs/faq/smart_device.md b/docs/faq/smart_device.md index 1c4a8919d981..c56457a626cb 100644 --- a/docs/faq/smart_device.md +++ b/docs/faq/smart_device.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Deep Learning in a Single File for Smart Devices diff --git a/docs/faq/visualize_graph.md b/docs/faq/visualize_graph.md index 4623346a5a81..adeb0e49136f 100644 --- a/docs/faq/visualize_graph.md +++ b/docs/faq/visualize_graph.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # How to visualize Neural Networks as computation graph diff --git a/docs/faq/why_mxnet.md b/docs/faq/why_mxnet.md index 00f21043cc41..140e853e2692 100644 --- a/docs/faq/why_mxnet.md +++ b/docs/faq/why_mxnet.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Why MXNet? diff --git a/docs/gluon/index.md b/docs/gluon/index.md index b136efbe0ae5..469222e46293 100644 --- a/docs/gluon/index.md +++ b/docs/gluon/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # About Gluon diff --git a/docs/index.md b/docs/index.md index 1ecdbde909d5..e7f314eb169b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # MXNet diff --git a/docs/install/amazonlinux_setup.md b/docs/install/amazonlinux_setup.md index baac53461ffd..a432d4815662 100644 --- a/docs/install/amazonlinux_setup.md +++ b/docs/install/amazonlinux_setup.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + diff --git a/docs/install/build_from_source.md b/docs/install/build_from_source.md index 0effd3b30979..fee4ae530277 100644 --- a/docs/install/build_from_source.md +++ b/docs/install/build_from_source.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Build MXNet from Source diff --git a/docs/install/c_plus_plus.md b/docs/install/c_plus_plus.md index f95bcc645746..ee21014bc5f1 100644 --- a/docs/install/c_plus_plus.md +++ b/docs/install/c_plus_plus.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + ## Build the C++ package The C++ package has the same prerequisites as the MXNet library. diff --git a/docs/install/centos_setup.md b/docs/install/centos_setup.md index a42c337c0f91..e01b68221bd9 100644 --- a/docs/install/centos_setup.md +++ b/docs/install/centos_setup.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Installing MXNet on CentOS and other non-Ubuntu Linux systems diff --git a/docs/install/download.md b/docs/install/download.md index 998336731d5f..725cf5eb72db 100644 --- a/docs/install/download.md +++ b/docs/install/download.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Source Download diff --git a/docs/install/index.md b/docs/install/index.md index c4da719e72d9..76f22e744075 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Installing MXNet diff --git a/docs/install/java_setup.md b/docs/install/java_setup.md index 39765730353a..569f439f8b02 100644 --- a/docs/install/java_setup.md +++ b/docs/install/java_setup.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Setup the MXNet Package for Java diff --git a/docs/install/osx_setup.md b/docs/install/osx_setup.md index e39c006b86ee..84e58282d766 100644 --- a/docs/install/osx_setup.md +++ b/docs/install/osx_setup.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Installing MXNet from source on OS X (Mac) diff --git a/docs/install/raspbian_setup.md b/docs/install/raspbian_setup.md index baac53461ffd..a432d4815662 100644 --- a/docs/install/raspbian_setup.md +++ b/docs/install/raspbian_setup.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + diff --git a/docs/install/scala_setup.md b/docs/install/scala_setup.md index a09fcc949e8f..bc069a14e6b3 100644 --- a/docs/install/scala_setup.md +++ b/docs/install/scala_setup.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Setup the MXNet Package for Scala diff --git a/docs/install/tx2_setup.md b/docs/install/tx2_setup.md index baac53461ffd..a432d4815662 100644 --- a/docs/install/tx2_setup.md +++ b/docs/install/tx2_setup.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + diff --git a/docs/install/ubuntu_setup.md b/docs/install/ubuntu_setup.md index 726888f377cb..bffbdb3e58f2 100644 --- a/docs/install/ubuntu_setup.md +++ b/docs/install/ubuntu_setup.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Installing MXNet on Ubuntu diff --git a/docs/install/validate_mxnet.md b/docs/install/validate_mxnet.md index 1dd4df89b149..c31463b63eee 100644 --- a/docs/install/validate_mxnet.md +++ b/docs/install/validate_mxnet.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Validate Your MXNet Installation diff --git a/docs/install/windows_setup.md b/docs/install/windows_setup.md index 383bca0b9778..3c3da5349235 100644 --- a/docs/install/windows_setup.md +++ b/docs/install/windows_setup.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Installing MXNet on Windows diff --git a/docs/model_zoo/index.md b/docs/model_zoo/index.md index 0a65a8d68ff8..cfe13caa0bf0 100644 --- a/docs/model_zoo/index.md +++ b/docs/model_zoo/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Model Zoo diff --git a/docs/tutorials/basic/data.md b/docs/tutorials/basic/data.md index e62ea4fa9737..00c87d0c37c5 100644 --- a/docs/tutorials/basic/data.md +++ b/docs/tutorials/basic/data.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Iterators - Loading data In this tutorial, we focus on how to feed data into a training or inference program. diff --git a/docs/tutorials/basic/index.md b/docs/tutorials/basic/index.md index aab7731342f7..faf6526fb824 100644 --- a/docs/tutorials/basic/index.md +++ b/docs/tutorials/basic/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/basic/module.md b/docs/tutorials/basic/module.md index 67f5d8aecd1f..64a1e9482ea8 100644 --- a/docs/tutorials/basic/module.md +++ b/docs/tutorials/basic/module.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Module - Neural network training and inference diff --git a/docs/tutorials/basic/ndarray.md b/docs/tutorials/basic/ndarray.md index 00cac64dd6ba..be5dbcf29e88 100644 --- a/docs/tutorials/basic/ndarray.md +++ b/docs/tutorials/basic/ndarray.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # NDArray - Imperative tensor operations on CPU/GPU diff --git a/docs/tutorials/basic/ndarray_indexing.md b/docs/tutorials/basic/ndarray_indexing.md index e06e5cb4482d..00a231351d7d 100644 --- a/docs/tutorials/basic/ndarray_indexing.md +++ b/docs/tutorials/basic/ndarray_indexing.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # NDArray Indexing - Array indexing features diff --git a/docs/tutorials/basic/reshape_transpose.md b/docs/tutorials/basic/reshape_transpose.md index 7407920b0aea..40bdb38f3ec2 100644 --- a/docs/tutorials/basic/reshape_transpose.md +++ b/docs/tutorials/basic/reshape_transpose.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + ## Difference between reshape and transpose operators diff --git a/docs/tutorials/basic/symbol.md b/docs/tutorials/basic/symbol.md index 5642bab74d64..cff12aca3c95 100644 --- a/docs/tutorials/basic/symbol.md +++ b/docs/tutorials/basic/symbol.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Symbol - Neural network graphs diff --git a/docs/tutorials/c++/basics.md b/docs/tutorials/c++/basics.md index a960b1817635..ddc5595cf3c0 100644 --- a/docs/tutorials/c++/basics.md +++ b/docs/tutorials/c++/basics.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Basics ====== diff --git a/docs/tutorials/c++/index.md b/docs/tutorials/c++/index.md index aab7731342f7..faf6526fb824 100644 --- a/docs/tutorials/c++/index.md +++ b/docs/tutorials/c++/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md b/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md index c408b8a06f52..ca4a75ef3b54 100644 --- a/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md +++ b/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet C++ API inference tutorial diff --git a/docs/tutorials/c++/subgraphAPI.md b/docs/tutorials/c++/subgraphAPI.md index a830bcbcf422..b834df8741b5 100644 --- a/docs/tutorials/c++/subgraphAPI.md +++ b/docs/tutorials/c++/subgraphAPI.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + ## Subgraph API diff --git a/docs/tutorials/control_flow/ControlFlowTutorial.md b/docs/tutorials/control_flow/ControlFlowTutorial.md index 173027b13bcf..88aa383cb011 100644 --- a/docs/tutorials/control_flow/ControlFlowTutorial.md +++ b/docs/tutorials/control_flow/ControlFlowTutorial.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Hybridize Gluon models with control flows. diff --git a/docs/tutorials/control_flow/index.md b/docs/tutorials/control_flow/index.md index aab7731342f7..faf6526fb824 100644 --- a/docs/tutorials/control_flow/index.md +++ b/docs/tutorials/control_flow/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/embedded/index.md b/docs/tutorials/embedded/index.md index aab7731342f7..faf6526fb824 100644 --- a/docs/tutorials/embedded/index.md +++ b/docs/tutorials/embedded/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/embedded/wine_detector.md b/docs/tutorials/embedded/wine_detector.md index b59a52bd8649..f0ae8273203e 100644 --- a/docs/tutorials/embedded/wine_detector.md +++ b/docs/tutorials/embedded/wine_detector.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Real-time Object Detection with MXNet On The Raspberry Pi diff --git a/docs/tutorials/gluon/autograd.md b/docs/tutorials/gluon/autograd.md index 495db34de825..4d8a042c9b03 100644 --- a/docs/tutorials/gluon/autograd.md +++ b/docs/tutorials/gluon/autograd.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Automatic differentiation diff --git a/docs/tutorials/gluon/custom_layer.md b/docs/tutorials/gluon/custom_layer.md index a45622d175cc..40ba0823a9d5 100644 --- a/docs/tutorials/gluon/custom_layer.md +++ b/docs/tutorials/gluon/custom_layer.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # How to write a custom layer in Apache MxNet Gluon API diff --git a/docs/tutorials/gluon/customop.md b/docs/tutorials/gluon/customop.md index 5552093484ba..eae0344c8702 100644 --- a/docs/tutorials/gluon/customop.md +++ b/docs/tutorials/gluon/customop.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Creating custom operators with numpy diff --git a/docs/tutorials/gluon/data_augmentation.md b/docs/tutorials/gluon/data_augmentation.md index ce631cea5fbf..356d335a8ca2 100644 --- a/docs/tutorials/gluon/data_augmentation.md +++ b/docs/tutorials/gluon/data_augmentation.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Methods of applying data augmentation (Gluon API) diff --git a/docs/tutorials/gluon/datasets.md b/docs/tutorials/gluon/datasets.md index 94b27bbbd331..c029124af8b6 100644 --- a/docs/tutorials/gluon/datasets.md +++ b/docs/tutorials/gluon/datasets.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Gluon `Dataset`s and `DataLoader` diff --git a/docs/tutorials/gluon/gluon.md b/docs/tutorials/gluon/gluon.md index 6cf27ff603d2..d9a6e93af689 100644 --- a/docs/tutorials/gluon/gluon.md +++ b/docs/tutorials/gluon/gluon.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Gluon - Neural network building blocks diff --git a/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md b/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md index 0394c84d1e02..42a65b352199 100644 --- a/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md +++ b/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Gluon: from experiment to deployment, an end to end tutorial diff --git a/docs/tutorials/gluon/gotchas_numpy_in_mxnet.md b/docs/tutorials/gluon/gotchas_numpy_in_mxnet.md index 4b2fa95ce885..1c285a41d76b 100644 --- a/docs/tutorials/gluon/gotchas_numpy_in_mxnet.md +++ b/docs/tutorials/gluon/gotchas_numpy_in_mxnet.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Gotchas using NumPy in Apache MXNet diff --git a/docs/tutorials/gluon/hybrid.md b/docs/tutorials/gluon/hybrid.md index c79f79692637..b7725386336e 100644 --- a/docs/tutorials/gluon/hybrid.md +++ b/docs/tutorials/gluon/hybrid.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Hybrid - Faster training and easy deployment diff --git a/docs/tutorials/gluon/index.md b/docs/tutorials/gluon/index.md index aab7731342f7..faf6526fb824 100644 --- a/docs/tutorials/gluon/index.md +++ b/docs/tutorials/gluon/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/gluon/info_gan.md b/docs/tutorials/gluon/info_gan.md index 2b8cdfe3fc4e..93fd6cb5e7fd 100644 --- a/docs/tutorials/gluon/info_gan.md +++ b/docs/tutorials/gluon/info_gan.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Image similarity search with InfoGAN diff --git a/docs/tutorials/gluon/learning_rate_finder.md b/docs/tutorials/gluon/learning_rate_finder.md index b4627f25bc39..30c66e302766 100644 --- a/docs/tutorials/gluon/learning_rate_finder.md +++ b/docs/tutorials/gluon/learning_rate_finder.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Learning Rate Finder diff --git a/docs/tutorials/gluon/learning_rate_schedules.md b/docs/tutorials/gluon/learning_rate_schedules.md index 3416ab44f992..46c79ebc249b 100644 --- a/docs/tutorials/gluon/learning_rate_schedules.md +++ b/docs/tutorials/gluon/learning_rate_schedules.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Learning Rate Schedules diff --git a/docs/tutorials/gluon/learning_rate_schedules_advanced.md b/docs/tutorials/gluon/learning_rate_schedules_advanced.md index 0d933997a2bb..287f70d535e8 100644 --- a/docs/tutorials/gluon/learning_rate_schedules_advanced.md +++ b/docs/tutorials/gluon/learning_rate_schedules_advanced.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Advanced Learning Rate Schedules diff --git a/docs/tutorials/gluon/logistic_regression_explained.md b/docs/tutorials/gluon/logistic_regression_explained.md index 93a93ccc9c32..57ed56eb8f4f 100644 --- a/docs/tutorials/gluon/logistic_regression_explained.md +++ b/docs/tutorials/gluon/logistic_regression_explained.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Logistic regression using Gluon API explained diff --git a/docs/tutorials/gluon/mnist.md b/docs/tutorials/gluon/mnist.md index aa1efccfb653..828bfd51f601 100644 --- a/docs/tutorials/gluon/mnist.md +++ b/docs/tutorials/gluon/mnist.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Hand-written Digit Recognition diff --git a/docs/tutorials/gluon/naming.md b/docs/tutorials/gluon/naming.md index 16383d309fc5..e667ad3cb79e 100644 --- a/docs/tutorials/gluon/naming.md +++ b/docs/tutorials/gluon/naming.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Naming of Gluon Parameter and Blocks diff --git a/docs/tutorials/gluon/ndarray.md b/docs/tutorials/gluon/ndarray.md index f4bae9494454..f1503aec18db 100644 --- a/docs/tutorials/gluon/ndarray.md +++ b/docs/tutorials/gluon/ndarray.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # NDArray - Scientific computing on CPU and GPU diff --git a/docs/tutorials/gluon/pretrained_models.md b/docs/tutorials/gluon/pretrained_models.md index 796fe05e731e..27ec1a95b8cd 100644 --- a/docs/tutorials/gluon/pretrained_models.md +++ b/docs/tutorials/gluon/pretrained_models.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Using pre-trained models in MXNet diff --git a/docs/tutorials/gluon/save_load_params.md b/docs/tutorials/gluon/save_load_params.md index feaf5eaf8547..ffebefdf80e1 100644 --- a/docs/tutorials/gluon/save_load_params.md +++ b/docs/tutorials/gluon/save_load_params.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Saving and Loading Gluon Models diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 562c66170a01..7e0ffaa3f72a 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Tutorials diff --git a/docs/tutorials/java/index.md b/docs/tutorials/java/index.md index aab7731342f7..faf6526fb824 100644 --- a/docs/tutorials/java/index.md +++ b/docs/tutorials/java/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/java/mxnet_java_on_intellij.md b/docs/tutorials/java/mxnet_java_on_intellij.md index b36ee3cb0ebc..ce84788c846e 100644 --- a/docs/tutorials/java/mxnet_java_on_intellij.md +++ b/docs/tutorials/java/mxnet_java_on_intellij.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Run MXNet Java Examples Using the IntelliJ IDE (macOS) diff --git a/docs/tutorials/java/ssd_inference.md b/docs/tutorials/java/ssd_inference.md index b61fe97036ef..1117bbdcfa5b 100644 --- a/docs/tutorials/java/ssd_inference.md +++ b/docs/tutorials/java/ssd_inference.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Multi Object Detection using pre-trained SSD Model via Java Inference APIs diff --git a/docs/tutorials/nlp/cnn.md b/docs/tutorials/nlp/cnn.md index 128f5caaae26..e671de3a1f57 100644 --- a/docs/tutorials/nlp/cnn.md +++ b/docs/tutorials/nlp/cnn.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Text Classification Using a Convolutional Neural Network on MXNet diff --git a/docs/tutorials/nlp/index.md b/docs/tutorials/nlp/index.md index aab7731342f7..faf6526fb824 100644 --- a/docs/tutorials/nlp/index.md +++ b/docs/tutorials/nlp/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/onnx/export_mxnet_to_onnx.md b/docs/tutorials/onnx/export_mxnet_to_onnx.md index 6190b9242a74..c0f8d92901f6 100644 --- a/docs/tutorials/onnx/export_mxnet_to_onnx.md +++ b/docs/tutorials/onnx/export_mxnet_to_onnx.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Exporting MXNet model to ONNX format diff --git a/docs/tutorials/onnx/fine_tuning_gluon.md b/docs/tutorials/onnx/fine_tuning_gluon.md index 70c1469afa94..dd0c0e93e862 100644 --- a/docs/tutorials/onnx/fine_tuning_gluon.md +++ b/docs/tutorials/onnx/fine_tuning_gluon.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Fine-tuning an ONNX model with MXNet/Gluon diff --git a/docs/tutorials/onnx/index.md b/docs/tutorials/onnx/index.md index aab7731342f7..faf6526fb824 100644 --- a/docs/tutorials/onnx/index.md +++ b/docs/tutorials/onnx/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/onnx/inference_on_onnx_model.md b/docs/tutorials/onnx/inference_on_onnx_model.md index f500956f72c6..f12e050fcc73 100644 --- a/docs/tutorials/onnx/inference_on_onnx_model.md +++ b/docs/tutorials/onnx/inference_on_onnx_model.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Running inference on MXNet/Gluon from an ONNX model diff --git a/docs/tutorials/onnx/super_resolution.md b/docs/tutorials/onnx/super_resolution.md index d9825090e8ee..db5932949f14 100644 --- a/docs/tutorials/onnx/super_resolution.md +++ b/docs/tutorials/onnx/super_resolution.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Importing an ONNX model into MXNet diff --git a/docs/tutorials/python/data_augmentation.md b/docs/tutorials/python/data_augmentation.md index 1034f621ba6f..45a4e3d8c86d 100644 --- a/docs/tutorials/python/data_augmentation.md +++ b/docs/tutorials/python/data_augmentation.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Methods of applying data augmentation (Module API) diff --git a/docs/tutorials/python/data_augmentation_with_masks.md b/docs/tutorials/python/data_augmentation_with_masks.md index 080708caedb1..66beece720a9 100644 --- a/docs/tutorials/python/data_augmentation_with_masks.md +++ b/docs/tutorials/python/data_augmentation_with_masks.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Data Augmentation with Masks diff --git a/docs/tutorials/python/index.md b/docs/tutorials/python/index.md index aab7731342f7..faf6526fb824 100644 --- a/docs/tutorials/python/index.md +++ b/docs/tutorials/python/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/python/kvstore.md b/docs/tutorials/python/kvstore.md index 593c240ad17c..42debab9b83e 100644 --- a/docs/tutorials/python/kvstore.md +++ b/docs/tutorials/python/kvstore.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Distributed Key-Value Store diff --git a/docs/tutorials/python/linear-regression.md b/docs/tutorials/python/linear-regression.md index 77ca7f9a7ceb..47d1acae9a14 100644 --- a/docs/tutorials/python/linear-regression.md +++ b/docs/tutorials/python/linear-regression.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Linear Regression diff --git a/docs/tutorials/python/matrix_factorization.md b/docs/tutorials/python/matrix_factorization.md index 62674e7ad646..cfe73a4856e0 100644 --- a/docs/tutorials/python/matrix_factorization.md +++ b/docs/tutorials/python/matrix_factorization.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Matrix Factorization diff --git a/docs/tutorials/python/mnist.md b/docs/tutorials/python/mnist.md index a8eee40e095e..9d641b36c202 100644 --- a/docs/tutorials/python/mnist.md +++ b/docs/tutorials/python/mnist.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Handwritten Digit Recognition diff --git a/docs/tutorials/python/predict_image.md b/docs/tutorials/python/predict_image.md index e02db165d7f6..0721abd6a731 100644 --- a/docs/tutorials/python/predict_image.md +++ b/docs/tutorials/python/predict_image.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Predict with pre-trained models diff --git a/docs/tutorials/python/profiler.md b/docs/tutorials/python/profiler.md index 1d6c0b2fc7f6..fe7611aa538f 100644 --- a/docs/tutorials/python/profiler.md +++ b/docs/tutorials/python/profiler.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Profiling MXNet Models diff --git a/docs/tutorials/python/types_of_data_augmentation.md b/docs/tutorials/python/types_of_data_augmentation.md index 344f3cd4d34a..63ad468f85f6 100644 --- a/docs/tutorials/python/types_of_data_augmentation.md +++ b/docs/tutorials/python/types_of_data_augmentation.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Types of Data Augmentation diff --git a/docs/tutorials/r/CallbackFunction.md b/docs/tutorials/r/CallbackFunction.md index f3156cd969e8..1258c527e456 100644 --- a/docs/tutorials/r/CallbackFunction.md +++ b/docs/tutorials/r/CallbackFunction.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Callback Function ====================================== diff --git a/docs/tutorials/r/CustomIterator.md b/docs/tutorials/r/CustomIterator.md index 62a618a51590..377838d93e46 100644 --- a/docs/tutorials/r/CustomIterator.md +++ b/docs/tutorials/r/CustomIterator.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Custom Iterator Tutorial ====================================== diff --git a/docs/tutorials/r/CustomLossFunction.md b/docs/tutorials/r/CustomLossFunction.md index 87ac40dafc63..6b60886bceba 100644 --- a/docs/tutorials/r/CustomLossFunction.md +++ b/docs/tutorials/r/CustomLossFunction.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Customized loss function ====================================== diff --git a/docs/tutorials/r/MultidimLstm.md b/docs/tutorials/r/MultidimLstm.md index 49a07361c9d4..5abd38d2e02c 100644 --- a/docs/tutorials/r/MultidimLstm.md +++ b/docs/tutorials/r/MultidimLstm.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + LSTM time series example ============================================= diff --git a/docs/tutorials/r/charRnnModel.md b/docs/tutorials/r/charRnnModel.md index f0d1f9e4d1fb..ed873d49214d 100644 --- a/docs/tutorials/r/charRnnModel.md +++ b/docs/tutorials/r/charRnnModel.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Character-level Language Model using RNN diff --git a/docs/tutorials/r/classifyRealImageWithPretrainedModel.md b/docs/tutorials/r/classifyRealImageWithPretrainedModel.md index 8a7daf87e4f1..1344285dac59 100644 --- a/docs/tutorials/r/classifyRealImageWithPretrainedModel.md +++ b/docs/tutorials/r/classifyRealImageWithPretrainedModel.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Classify Images with a PreTrained Model ================================================= diff --git a/docs/tutorials/r/fiveMinutesNeuralNetwork.md b/docs/tutorials/r/fiveMinutesNeuralNetwork.md index 56688a65338e..39775b3ab640 100644 --- a/docs/tutorials/r/fiveMinutesNeuralNetwork.md +++ b/docs/tutorials/r/fiveMinutesNeuralNetwork.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Develop a Neural Network with MXNet in Five Minutes ============================================= diff --git a/docs/tutorials/r/index.md b/docs/tutorials/r/index.md index 6ab039561886..13db9694864d 100644 --- a/docs/tutorials/r/index.md +++ b/docs/tutorials/r/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # R Tutorials diff --git a/docs/tutorials/r/mnistCompetition.md b/docs/tutorials/r/mnistCompetition.md index 95efbfd6d7af..f59aabd77905 100644 --- a/docs/tutorials/r/mnistCompetition.md +++ b/docs/tutorials/r/mnistCompetition.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Handwritten Digits Classification Competition ============================================= diff --git a/docs/tutorials/r/ndarray.md b/docs/tutorials/r/ndarray.md index 30f5d887a3b6..e935a5052cf3 100644 --- a/docs/tutorials/r/ndarray.md +++ b/docs/tutorials/r/ndarray.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # NDArray: Vectorized Tensor Computations on CPUs and GPUs diff --git a/docs/tutorials/r/symbol.md b/docs/tutorials/r/symbol.md index 0ab560856bb4..d60c5fbfbc59 100644 --- a/docs/tutorials/r/symbol.md +++ b/docs/tutorials/r/symbol.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Symbol and Automatic Differentiation diff --git a/docs/tutorials/scala/char_lstm.md b/docs/tutorials/scala/char_lstm.md index fafd5d5a0bd9..972661bc81ef 100644 --- a/docs/tutorials/scala/char_lstm.md +++ b/docs/tutorials/scala/char_lstm.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Developing a Character-level Language model diff --git a/docs/tutorials/scala/index.md b/docs/tutorials/scala/index.md index 55e41e428d38..1e03345a1e45 100644 --- a/docs/tutorials/scala/index.md +++ b/docs/tutorials/scala/index.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet-Scala Tutorials diff --git a/docs/tutorials/scala/mnist.md b/docs/tutorials/scala/mnist.md index 1f05fb464fb0..89bc203231f7 100644 --- a/docs/tutorials/scala/mnist.md +++ b/docs/tutorials/scala/mnist.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Handwritten Digit Recognition diff --git a/docs/tutorials/scala/mxnet_scala_on_intellij.md b/docs/tutorials/scala/mxnet_scala_on_intellij.md index cae70567de4d..f1d15cb3c5b5 100644 --- a/docs/tutorials/scala/mxnet_scala_on_intellij.md +++ b/docs/tutorials/scala/mxnet_scala_on_intellij.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Run MXNet Scala Examples Using the IntelliJ IDE (macOS) diff --git a/docs/tutorials/sparse/csr.md b/docs/tutorials/sparse/csr.md index f1718687cfb0..25de1f716045 100644 --- a/docs/tutorials/sparse/csr.md +++ b/docs/tutorials/sparse/csr.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # CSRNDArray - NDArray in Compressed Sparse Row Storage Format diff --git a/docs/tutorials/sparse/index.md b/docs/tutorials/sparse/index.md index aab7731342f7..faf6526fb824 100644 --- a/docs/tutorials/sparse/index.md +++ b/docs/tutorials/sparse/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/sparse/row_sparse.md b/docs/tutorials/sparse/row_sparse.md index 2986c4d3ab8f..f89de3d5b75e 100644 --- a/docs/tutorials/sparse/row_sparse.md +++ b/docs/tutorials/sparse/row_sparse.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # RowSparseNDArray - NDArray for Sparse Gradient Updates diff --git a/docs/tutorials/sparse/train.md b/docs/tutorials/sparse/train.md index d22fbe9e8f0c..2f315cfa1010 100644 --- a/docs/tutorials/sparse/train.md +++ b/docs/tutorials/sparse/train.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Train a Linear Regression Model with Sparse Symbols diff --git a/docs/tutorials/speech_recognition/ctc.md b/docs/tutorials/speech_recognition/ctc.md index 3f1b8d4ff9f7..948e681de52f 100644 --- a/docs/tutorials/speech_recognition/ctc.md +++ b/docs/tutorials/speech_recognition/ctc.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Connectionist Temporal Classification diff --git a/docs/tutorials/speech_recognition/index.md b/docs/tutorials/speech_recognition/index.md index aab7731342f7..faf6526fb824 100644 --- a/docs/tutorials/speech_recognition/index.md +++ b/docs/tutorials/speech_recognition/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/tensorrt/index.md b/docs/tutorials/tensorrt/index.md index d3dcaeb9db84..bc10874886ef 100644 --- a/docs/tutorials/tensorrt/index.md +++ b/docs/tutorials/tensorrt/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/tensorrt/inference_with_trt.md b/docs/tutorials/tensorrt/inference_with_trt.md index 77f82a7a1039..ff3cddf3d574 100644 --- a/docs/tutorials/tensorrt/inference_with_trt.md +++ b/docs/tutorials/tensorrt/inference_with_trt.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Optimizing Deep Learning Computation Graphs with TensorRT diff --git a/docs/tutorials/unsupervised_learning/gan.md b/docs/tutorials/unsupervised_learning/gan.md index 9c50fcfdf15b..ca0fb15e01c5 100644 --- a/docs/tutorials/unsupervised_learning/gan.md +++ b/docs/tutorials/unsupervised_learning/gan.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Generative Adversarial Network (GAN) diff --git a/docs/tutorials/unsupervised_learning/index.md b/docs/tutorials/unsupervised_learning/index.md index aab7731342f7..faf6526fb824 100644 --- a/docs/tutorials/unsupervised_learning/index.md +++ b/docs/tutorials/unsupervised_learning/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/vision/cnn_visualization.md b/docs/tutorials/vision/cnn_visualization.md index f528d35bfd32..1eb89e104264 100644 --- a/docs/tutorials/vision/cnn_visualization.md +++ b/docs/tutorials/vision/cnn_visualization.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Visualizing Decisions of Convolutional Neural Networks diff --git a/docs/tutorials/vision/index.md b/docs/tutorials/vision/index.md index aab7731342f7..faf6526fb824 100644 --- a/docs/tutorials/vision/index.md +++ b/docs/tutorials/vision/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Tutorials diff --git a/docs/tutorials/vision/large_scale_classification.md b/docs/tutorials/vision/large_scale_classification.md index 5fa87b14821d..36a0ff6c9daa 100644 --- a/docs/tutorials/vision/large_scale_classification.md +++ b/docs/tutorials/vision/large_scale_classification.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Large Scale Image Classification diff --git a/example/README.md b/example/README.md index 5594e3ed0f3d..cd49cf9ff9af 100644 --- a/example/README.md +++ b/example/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Examples diff --git a/example/adversary/README.md b/example/adversary/README.md index 7f5158c03bbf..d7c2f46d9efb 100644 --- a/example/adversary/README.md +++ b/example/adversary/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Adversarial examples diff --git a/example/autoencoder/README.md b/example/autoencoder/README.md index 15f4851412af..9db075e680f0 100644 --- a/example/autoencoder/README.md +++ b/example/autoencoder/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Example of a Convolutional Autoencoder diff --git a/example/autoencoder/variational_autoencoder/README.md b/example/autoencoder/variational_autoencoder/README.md index 07d099848b46..2a7a4966f596 100644 --- a/example/autoencoder/variational_autoencoder/README.md +++ b/example/autoencoder/variational_autoencoder/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Variational Auto Encoder(VAE) ============================= diff --git a/example/bayesian-methods/README.md b/example/bayesian-methods/README.md index 56e4b1dd92eb..7604495eb151 100644 --- a/example/bayesian-methods/README.md +++ b/example/bayesian-methods/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Bayesian Methods ================ diff --git a/example/bi-lstm-sort/README.md b/example/bi-lstm-sort/README.md index e08db18a9bcf..e26e2b02de63 100644 --- a/example/bi-lstm-sort/README.md +++ b/example/bi-lstm-sort/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Bidirectionnal LSTM to sort an array. diff --git a/example/caffe/README.md b/example/caffe/README.md index 6541d4dacc04..a497176a996f 100644 --- a/example/caffe/README.md +++ b/example/caffe/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # How to use Caffe operator in MXNet diff --git a/example/capsnet/README.md b/example/capsnet/README.md index 4c05781a3971..9319ea3b43bc 100644 --- a/example/capsnet/README.md +++ b/example/capsnet/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + **CapsNet-MXNet** ========================================= diff --git a/example/captcha/README.md b/example/captcha/README.md index 3936178ebd3e..7743997e90bd 100644 --- a/example/captcha/README.md +++ b/example/captcha/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + This is the R version of [captcha recognition](http://blog.xlvector.net/2016-05/mxnet-ocr-cnn/) example by xlvector and it can be used as an example of multi-label training. For a captcha below, we consider it as an image with 4 labels and train a CNN over the data set. diff --git a/example/cnn_chinese_text_classification/README.md b/example/cnn_chinese_text_classification/README.md index 5bdaecb8c155..e28a0ec9ac47 100644 --- a/example/cnn_chinese_text_classification/README.md +++ b/example/cnn_chinese_text_classification/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Implementing CNN + Highway Network for Chinese Text Classification in MXNet ============ diff --git a/example/cnn_text_classification/README.md b/example/cnn_text_classification/README.md index c04a22cde103..f4ebc43afa96 100644 --- a/example/cnn_text_classification/README.md +++ b/example/cnn_text_classification/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Implementing CNN for Text Classification in MXNet ============ diff --git a/example/ctc/README.md b/example/ctc/README.md index 96373db002d8..e64c6e633b2f 100644 --- a/example/ctc/README.md +++ b/example/ctc/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Connectionist Temporal Classification diff --git a/example/deep-embedded-clustering/README.md b/example/deep-embedded-clustering/README.md index 4e626a6bac81..f3b532d0023d 100644 --- a/example/deep-embedded-clustering/README.md +++ b/example/deep-embedded-clustering/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # DEC Implementation This is based on the paper `Unsupervised deep embedding for clustering analysis` by Junyuan Xie, Ross Girshick, and Ali Farhadi diff --git a/example/distributed_training/README.md b/example/distributed_training/README.md index ba09cb91aede..d1bd11738cec 100644 --- a/example/distributed_training/README.md +++ b/example/distributed_training/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Distributed Training using Gluon diff --git a/example/dsd/README.md b/example/dsd/README.md index bffacce8cabc..4a2a02cddbfa 100644 --- a/example/dsd/README.md +++ b/example/dsd/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + DSD Training ============ diff --git a/example/fcn-xs/README.md b/example/fcn-xs/README.md index 98d8edab6f93..2d81f71ed68b 100644 --- a/example/fcn-xs/README.md +++ b/example/fcn-xs/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + FCN-xs EXAMPLE -------------- diff --git a/example/gluon/audio/urban_sounds/README.md b/example/gluon/audio/urban_sounds/README.md index 18593272f144..c593f28dbee0 100644 --- a/example/gluon/audio/urban_sounds/README.md +++ b/example/gluon/audio/urban_sounds/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Urban Sounds Classification in MXNet Gluon diff --git a/example/gluon/dc_gan/README.md b/example/gluon/dc_gan/README.md index 1d91d91a3452..fd41d198a69d 100644 --- a/example/gluon/dc_gan/README.md +++ b/example/gluon/dc_gan/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # DCGAN in MXNet diff --git a/example/gluon/embedding_learning/README.md b/example/gluon/embedding_learning/README.md index de11ea0db072..ce1fb536a859 100644 --- a/example/gluon/embedding_learning/README.md +++ b/example/gluon/embedding_learning/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Image Embedding Learning diff --git a/example/gluon/lipnet/README.md b/example/gluon/lipnet/README.md index 70eda1765bd3..894384bc152d 100644 --- a/example/gluon/lipnet/README.md +++ b/example/gluon/lipnet/README.md @@ -1,3 +1,22 @@ + + # LipNet: End-to-End Sentence-level Lipreading --- diff --git a/example/gluon/sn_gan/README.md b/example/gluon/sn_gan/README.md index 60327bc959a6..054416fced09 100644 --- a/example/gluon/sn_gan/README.md +++ b/example/gluon/sn_gan/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Spectral Normalization GAN diff --git a/example/gluon/style_transfer/README.md b/example/gluon/style_transfer/README.md index 1162e7241053..c182f3f8fce8 100644 --- a/example/gluon/style_transfer/README.md +++ b/example/gluon/style_transfer/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet-Gluon-Style-Transfer diff --git a/example/gluon/tree_lstm/README.md b/example/gluon/tree_lstm/README.md index 93d6289f42af..8e3b385b77b0 100644 --- a/example/gluon/tree_lstm/README.md +++ b/example/gluon/tree_lstm/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Tree-Structured Long Short-Term Memory Networks diff --git a/example/gluon/word_language_model/README.md b/example/gluon/word_language_model/README.md index 7c9b74ad3379..04a4a4ac70b8 100644 --- a/example/gluon/word_language_model/README.md +++ b/example/gluon/word_language_model/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Word-level language modeling RNN diff --git a/example/kaggle-ndsb1/README.md b/example/kaggle-ndsb1/README.md index 54d33b8b1df2..804ed9474dc9 100644 --- a/example/kaggle-ndsb1/README.md +++ b/example/kaggle-ndsb1/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + Tutorial for Kaggle NDSB-1 ----- diff --git a/example/kaggle-ndsb2/README.md b/example/kaggle-ndsb2/README.md index d1e29d4f86bb..62a751b69302 100644 --- a/example/kaggle-ndsb2/README.md +++ b/example/kaggle-ndsb2/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # End-to-End Deep Learning Tutorial for Kaggle NDSB-II diff --git a/example/model-parallel/matrix_factorization/README.md b/example/model-parallel/matrix_factorization/README.md index ee5384565d45..94ec91d0e8b8 100644 --- a/example/model-parallel/matrix_factorization/README.md +++ b/example/model-parallel/matrix_factorization/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Model Parallel Matrix Factorization =================================== diff --git a/example/module/README.md b/example/module/README.md index 8dd3d6d87c7c..24c917d5c8bc 100644 --- a/example/module/README.md +++ b/example/module/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Module Usage Example diff --git a/example/multi-task/README.md b/example/multi-task/README.md index 4ca744189221..a110fd0bd7cf 100644 --- a/example/multi-task/README.md +++ b/example/multi-task/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Mulit-task learning example diff --git a/example/multivariate_time_series/README.md b/example/multivariate_time_series/README.md index 6c9e1a451f10..c3db6c130759 100644 --- a/example/multivariate_time_series/README.md +++ b/example/multivariate_time_series/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # LSTNet diff --git a/example/named_entity_recognition/README.md b/example/named_entity_recognition/README.md index 897ea4221b36..eaa358d4f18c 100644 --- a/example/named_entity_recognition/README.md +++ b/example/named_entity_recognition/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + ## Goal diff --git a/example/nce-loss/README.md b/example/nce-loss/README.md index 4a847e420732..26b028001f85 100644 --- a/example/nce-loss/README.md +++ b/example/nce-loss/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Examples of NCE Loss diff --git a/example/neural-style/README.md b/example/neural-style/README.md index 0f9545cf65b6..e8e69ccc2194 100644 --- a/example/neural-style/README.md +++ b/example/neural-style/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Neural art diff --git a/example/neural-style/end_to_end/README.md b/example/neural-style/end_to_end/README.md index 0c38f427c300..209d98df7254 100644 --- a/example/neural-style/end_to_end/README.md +++ b/example/neural-style/end_to_end/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # End to End Neural Art diff --git a/example/numpy-ops/README.md b/example/numpy-ops/README.md index a1ed1a5573e4..61bf8365bd18 100644 --- a/example/numpy-ops/README.md +++ b/example/numpy-ops/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Training with Custom Operators in Python diff --git a/example/profiler/README.md b/example/profiler/README.md index 14668fdbe0e5..255ad8f65579 100644 --- a/example/profiler/README.md +++ b/example/profiler/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Profiler Examples diff --git a/example/rcnn/README.md b/example/rcnn/README.md index 737ef4418914..e773089f7219 100644 --- a/example/rcnn/README.md +++ b/example/rcnn/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Faster R-CNN in MXNet diff --git a/example/recommenders/README.md b/example/recommenders/README.md index 2534074268e8..806e51f24c8d 100644 --- a/example/recommenders/README.md +++ b/example/recommenders/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Recommender Systems diff --git a/example/reinforcement-learning/a3c/README.md b/example/reinforcement-learning/a3c/README.md index 1a8b4f3ba7a1..55ade6a279f4 100644 --- a/example/reinforcement-learning/a3c/README.md +++ b/example/reinforcement-learning/a3c/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # A3C Implementation This is an attempt to implement the A3C algorithm in paper Asynchronous Methods for Deep Reinforcement Learning. diff --git a/example/reinforcement-learning/ddpg/README.md b/example/reinforcement-learning/ddpg/README.md index c3040434823a..69e00d982f98 100644 --- a/example/reinforcement-learning/ddpg/README.md +++ b/example/reinforcement-learning/ddpg/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # mx-DDPG MXNet Implementation of DDPG diff --git a/example/reinforcement-learning/parallel_actor_critic/README.md b/example/reinforcement-learning/parallel_actor_critic/README.md index 767c67a7d72b..5a442079b3a2 100644 --- a/example/reinforcement-learning/parallel_actor_critic/README.md +++ b/example/reinforcement-learning/parallel_actor_critic/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # 'Parallel Advantage-Actor Critic' Implementation diff --git a/example/restricted-boltzmann-machine/README.md b/example/restricted-boltzmann-machine/README.md index 30ee8cfb9c3d..d1609049c137 100644 --- a/example/restricted-boltzmann-machine/README.md +++ b/example/restricted-boltzmann-machine/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Restricted Boltzmann machine (RBM) diff --git a/example/rnn/README.md b/example/rnn/README.md index a819dfa1097b..cbafc19e2ef0 100644 --- a/example/rnn/README.md +++ b/example/rnn/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Recurrent Neural Network Examples =========== diff --git a/example/rnn/bucketing/README.md b/example/rnn/bucketing/README.md index e06d9f64da27..707370af5a96 100644 --- a/example/rnn/bucketing/README.md +++ b/example/rnn/bucketing/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + RNN Example =========== diff --git a/example/rnn/old/README.md b/example/rnn/old/README.md index 806f50b834de..b199c7770114 100644 --- a/example/rnn/old/README.md +++ b/example/rnn/old/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + RNN Example =========== diff --git a/example/rnn/word_lm/README.md b/example/rnn/word_lm/README.md index dbb7832e15dc..9f9e1b75b51e 100644 --- a/example/rnn/word_lm/README.md +++ b/example/rnn/word_lm/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + Word Level Language Modeling =========== diff --git a/example/sparse/factorization_machine/README.md b/example/sparse/factorization_machine/README.md index 42b93741fedb..0023b33a1413 100644 --- a/example/sparse/factorization_machine/README.md +++ b/example/sparse/factorization_machine/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Factorization Machine =========== diff --git a/example/sparse/linear_classification/README.md b/example/sparse/linear_classification/README.md index 38fde454f1b2..05674cd0c277 100644 --- a/example/sparse/linear_classification/README.md +++ b/example/sparse/linear_classification/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Linear Classification Using Sparse Matrix Multiplication =========== diff --git a/example/sparse/matrix_factorization/README.md b/example/sparse/matrix_factorization/README.md index d11219771012..88ae7b8e7678 100644 --- a/example/sparse/matrix_factorization/README.md +++ b/example/sparse/matrix_factorization/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + Matrix Factorization w/ Sparse Embedding =========== diff --git a/example/sparse/wide_deep/README.md b/example/sparse/wide_deep/README.md index 80924274e032..769d72365402 100644 --- a/example/sparse/wide_deep/README.md +++ b/example/sparse/wide_deep/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + ## Wide and Deep Learning diff --git a/example/speech_recognition/README.md b/example/speech_recognition/README.md index 3dad016a48d8..d2125f6578c1 100644 --- a/example/speech_recognition/README.md +++ b/example/speech_recognition/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + **deepSpeech.mxnet: Rich Speech Example** ========================================= diff --git a/example/ssd/README.md b/example/ssd/README.md index 753c967aeaf1..6d4caa481bd7 100644 --- a/example/ssd/README.md +++ b/example/ssd/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # SSD: Single Shot MultiBox Object Detector diff --git a/example/ssd/dataset/pycocotools/README.md b/example/ssd/dataset/pycocotools/README.md index 0e10d4080aac..ed4411425f8c 100644 --- a/example/ssd/dataset/pycocotools/README.md +++ b/example/ssd/dataset/pycocotools/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + This is a modified version of https://github.com/pdollar/coco python API. No `make` is required, but this will not support mask functions. diff --git a/example/ssd/model/README.md b/example/ssd/model/README.md index 0ee70c3d1fff..7c77ee2475e8 100644 --- a/example/ssd/model/README.md +++ b/example/ssd/model/README.md @@ -1,20 +1,18 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + #### This is the default directory to store all the models, including `*.params` and `*.json` diff --git a/example/ssd/symbol/README.md b/example/ssd/symbol/README.md index 67f64f584f8b..d577b7067c92 100644 --- a/example/ssd/symbol/README.md +++ b/example/ssd/symbol/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + ## How to compose SSD network on top of mainstream classification networks diff --git a/example/ssd/tools/caffe_converter/README.md b/example/ssd/tools/caffe_converter/README.md index 9eb59812e207..3d4ab13f4694 100644 --- a/example/ssd/tools/caffe_converter/README.md +++ b/example/ssd/tools/caffe_converter/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Convert Caffe Model to Mxnet Format diff --git a/example/stochastic-depth/README.md b/example/stochastic-depth/README.md index a5dfcd9c8e42..4a41199893ac 100644 --- a/example/stochastic-depth/README.md +++ b/example/stochastic-depth/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Stochastic Depth ================ diff --git a/example/svm_mnist/README.md b/example/svm_mnist/README.md index ffcf5c38bbd7..8d878fb76859 100644 --- a/example/svm_mnist/README.md +++ b/example/svm_mnist/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Use case with Support Vector Machine diff --git a/example/svrg_module/README.md b/example/svrg_module/README.md index 624b50d60f12..f77ecf8fd39d 100644 --- a/example/svrg_module/README.md +++ b/example/svrg_module/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + ## SVRGModule Example diff --git a/example/vae-gan/README.md b/example/vae-gan/README.md index 5d7ae272b32b..17e5b0e68e76 100644 --- a/example/vae-gan/README.md +++ b/example/vae-gan/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # VAE-GAN in MXNet diff --git a/julia/LICENSE.md b/julia/LICENSE.md index 4161166b7280..3e2c5a2673b8 100644 --- a/julia/LICENSE.md +++ b/julia/LICENSE.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + The MXNet.jl package is licensed under version 2.0 of the Apache License: diff --git a/julia/NEWS.md b/julia/NEWS.md index 1c8cda7176c6..4a6c1a210e17 100644 --- a/julia/NEWS.md +++ b/julia/NEWS.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # v1.5.0 (#TBD) diff --git a/julia/README-DEV.md b/julia/README-DEV.md index 9edb8309c0c8..25a6c6d93ab9 100644 --- a/julia/README-DEV.md +++ b/julia/README-DEV.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Workflow for making a release diff --git a/julia/README.md b/julia/README.md index 97a52ae9a4e6..43b8deb0666a 100644 --- a/julia/README.md +++ b/julia/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet diff --git a/julia/docs/src/api.md b/julia/docs/src/api.md index 92c04581c7e8..60cb0831d1bf 100644 --- a/julia/docs/src/api.md +++ b/julia/docs/src/api.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # API Documentation diff --git a/julia/docs/src/api/callback.md b/julia/docs/src/api/callback.md index 327014d7dbfc..5a35e5047120 100644 --- a/julia/docs/src/api/callback.md +++ b/julia/docs/src/api/callback.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Callback in training diff --git a/julia/docs/src/api/context.md b/julia/docs/src/api/context.md index e82504663b06..2daabe2db41b 100644 --- a/julia/docs/src/api/context.md +++ b/julia/docs/src/api/context.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Context diff --git a/julia/docs/src/api/executor.md b/julia/docs/src/api/executor.md index 81ce7fa17ef7..c3037dfff60b 100644 --- a/julia/docs/src/api/executor.md +++ b/julia/docs/src/api/executor.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Executor diff --git a/julia/docs/src/api/initializer.md b/julia/docs/src/api/initializer.md index 8f55b6e1da51..b2515263f93a 100644 --- a/julia/docs/src/api/initializer.md +++ b/julia/docs/src/api/initializer.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Initializer diff --git a/julia/docs/src/api/io.md b/julia/docs/src/api/io.md index bd53d2ca667e..34ad3c42bce7 100644 --- a/julia/docs/src/api/io.md +++ b/julia/docs/src/api/io.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Data Providers diff --git a/julia/docs/src/api/kvstore.md b/julia/docs/src/api/kvstore.md index a8407b54bd34..e6bf852b2f43 100644 --- a/julia/docs/src/api/kvstore.md +++ b/julia/docs/src/api/kvstore.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Key-Value Store diff --git a/julia/docs/src/api/metric.md b/julia/docs/src/api/metric.md index a31bc92e49e6..163f27a5b397 100644 --- a/julia/docs/src/api/metric.md +++ b/julia/docs/src/api/metric.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Evaluation Metrics diff --git a/julia/docs/src/api/model.md b/julia/docs/src/api/model.md index 8240aeb022bb..63137532de6a 100644 --- a/julia/docs/src/api/model.md +++ b/julia/docs/src/api/model.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Model diff --git a/julia/docs/src/api/ndarray.md b/julia/docs/src/api/ndarray.md index 41161d547881..8cc4948e4dde 100644 --- a/julia/docs/src/api/ndarray.md +++ b/julia/docs/src/api/ndarray.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # NDArray API diff --git a/julia/docs/src/api/nn-factory.md b/julia/docs/src/api/nn-factory.md index d8106eff158f..70ecfd2f0157 100644 --- a/julia/docs/src/api/nn-factory.md +++ b/julia/docs/src/api/nn-factory.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Neural Network Factory diff --git a/julia/docs/src/api/symbolic-node.md b/julia/docs/src/api/symbolic-node.md index c23b67602627..b4b1c0167e5e 100644 --- a/julia/docs/src/api/symbolic-node.md +++ b/julia/docs/src/api/symbolic-node.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Symbolic API diff --git a/julia/docs/src/api/visualize.md b/julia/docs/src/api/visualize.md index 843922420a30..e401a888cc81 100644 --- a/julia/docs/src/api/visualize.md +++ b/julia/docs/src/api/visualize.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Network Visualization diff --git a/julia/docs/src/index.md b/julia/docs/src/index.md index 8274c712e549..11a77670eae3 100644 --- a/julia/docs/src/index.md +++ b/julia/docs/src/index.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # MXNet Documentation diff --git a/julia/docs/src/tutorial/char-lstm.md b/julia/docs/src/tutorial/char-lstm.md index 53a371e028fe..3961744f282d 100644 --- a/julia/docs/src/tutorial/char-lstm.md +++ b/julia/docs/src/tutorial/char-lstm.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Generating Random Sentence with LSTM RNN ======================================== diff --git a/julia/docs/src/tutorial/mnist.md b/julia/docs/src/tutorial/mnist.md index 96423266db1b..cc5267071f11 100644 --- a/julia/docs/src/tutorial/mnist.md +++ b/julia/docs/src/tutorial/mnist.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Digit Recognition on MNIST ========================== diff --git a/julia/docs/src/user-guide/faq.md b/julia/docs/src/user-guide/faq.md index d288ad0b3b8e..2799584f5472 100644 --- a/julia/docs/src/user-guide/faq.md +++ b/julia/docs/src/user-guide/faq.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + FAQ === diff --git a/julia/docs/src/user-guide/install.md b/julia/docs/src/user-guide/install.md index 52628de7a255..d1a90a808507 100644 --- a/julia/docs/src/user-guide/install.md +++ b/julia/docs/src/user-guide/install.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Installation Guide ================== diff --git a/julia/docs/src/user-guide/overview.md b/julia/docs/src/user-guide/overview.md index f0189f7bcd2e..d1d32e80ebd7 100644 --- a/julia/docs/src/user-guide/overview.md +++ b/julia/docs/src/user-guide/overview.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Overview diff --git a/julia/examples/char-lstm/README.md b/julia/examples/char-lstm/README.md index ac77e15b131f..ac745dd4cc41 100644 --- a/julia/examples/char-lstm/README.md +++ b/julia/examples/char-lstm/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # LSTM char-rnn diff --git a/julia/plugins/README.md b/julia/plugins/README.md index c5ca926ca0ac..f9925cbdfbde 100644 --- a/julia/plugins/README.md +++ b/julia/plugins/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Plugins of MXNet.jl diff --git a/matlab/README.md b/matlab/README.md index 13a83922d915..d5ef5d09fc8d 100644 --- a/matlab/README.md +++ b/matlab/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MATLAB binding for MXNet diff --git a/perl-package/AI-MXNet/examples/gluon/style_transfer/README.md b/perl-package/AI-MXNet/examples/gluon/style_transfer/README.md index 3da1f97b517c..9be9b8ad023e 100644 --- a/perl-package/AI-MXNet/examples/gluon/style_transfer/README.md +++ b/perl-package/AI-MXNet/examples/gluon/style_transfer/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + This directory provides AI::MXNet Implementation of MSG-Net real time style transfer, https://arxiv.org/abs/1703.06953 diff --git a/perl-package/AI-MXNet/examples/sparse/matrix_factorization/README.md b/perl-package/AI-MXNet/examples/sparse/matrix_factorization/README.md index e4c89db760f4..3eb1bab508e5 100644 --- a/perl-package/AI-MXNet/examples/sparse/matrix_factorization/README.md +++ b/perl-package/AI-MXNet/examples/sparse/matrix_factorization/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + Matrix Factorization w/ Sparse Embedding =========== diff --git a/perl-package/AI-MXNet/examples/sparse/wide_deep/README.md b/perl-package/AI-MXNet/examples/sparse/wide_deep/README.md index fc3192623de4..4a01da4254e6 100644 --- a/perl-package/AI-MXNet/examples/sparse/wide_deep/README.md +++ b/perl-package/AI-MXNet/examples/sparse/wide_deep/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + ## Wide and Deep Learning diff --git a/perl-package/README.md b/perl-package/README.md index 20ce7e635e6f..893a11d19f88 100644 --- a/perl-package/README.md +++ b/perl-package/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + [Perl API](https://mxnet.incubator.apache.org/api/perl/index.html) [![GitHub license](http://dmlc.github.io/img/apache2.svg)](../LICENSE) diff --git a/plugin/caffe/README.md b/plugin/caffe/README.md index 6541d4dacc04..a497176a996f 100644 --- a/plugin/caffe/README.md +++ b/plugin/caffe/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # How to use Caffe operator in MXNet diff --git a/python/README.md b/python/README.md index 4e180360f674..396d112f687a 100644 --- a/python/README.md +++ b/python/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + MXNet Python Package ==================== diff --git a/python/minpy/README.md b/python/minpy/README.md new file mode 100644 index 000000000000..0278ca1c00fb --- /dev/null +++ b/python/minpy/README.md @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + +MXNet Python Package +==================== + +This is the WIP directory for MinPy project. diff --git a/scala-package/README.md b/scala-package/README.md index 7dd5f5ea0680..8322ab2a237f 100644 --- a/scala-package/README.md +++ b/scala-package/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + MXNet Package for Scala/Java ===== diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/objectdetector/README.md b/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/objectdetector/README.md index b6c92c1204fa..8a9ed3e1736b 100644 --- a/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/objectdetector/README.md +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/objectdetector/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Single Shot Multi Object Detection using Java Inference API diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/predictor/README.md b/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/predictor/README.md index cfad6a4e9a6e..09189cb83268 100644 --- a/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/predictor/README.md +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/predictor/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Image Classification using Java Predictor diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md index efeab0a188cf..d45469ac5296 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Benchmarking Scala Inference APIs diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md index ae2b68002e3c..3b28e3ef4463 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # CNN Text Classification Example for Scala This is the example using Scala type-safe api doing CNN text classification. diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md index bf2429399e94..a3952aabfb44 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Custom Operator Example for Scala This is the example using Custom Operator for type-safe api of Scala. diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/gan/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/gan/README.md index fd477c56f54e..a4536a7662e4 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/gan/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/gan/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # GAN MNIST Example for Scala This is the GAN MNIST Training Example implemented for Scala type-safe api diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/imclassification/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/imclassification/README.md index 55e065e1f493..e533130aa71c 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/imclassification/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/imclassification/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Image Classification Models diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md index 5e8a51789300..e32a66bca832 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Image Classification diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md index e3190b2fbcbf..e5d3bbee0490 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Single Shot Multi Object Detection using Scala Inference API diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/neuralstyle/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/neuralstyle/README.md index 2d39a9c67733..0dc4fb892691 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/neuralstyle/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/neuralstyle/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Neural Style Example for Scala diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/README.md index dea2c12667e9..2ef2f47c7983 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # RNN Example for MXNet Scala This folder contains the following examples writing in new Scala type-safe API: diff --git a/scala-package/memory-management.md b/scala-package/memory-management.md index e9a0c71ded02..b97bcbcb9026 100644 --- a/scala-package/memory-management.md +++ b/scala-package/memory-management.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # JVM Memory Management The Scala and Java bindings of Apache MXNet use native memory (memory from the C++ heap in either RAM or GPU memory) for most of the MXNet objects such as NDArray, Symbol, Executor, KVStore, Data Iterators, etc. diff --git a/scala-package/mxnet-demo/java-demo/README.md b/scala-package/mxnet-demo/java-demo/README.md index 3d5041c228f6..760219343ed2 100644 --- a/scala-package/mxnet-demo/java-demo/README.md +++ b/scala-package/mxnet-demo/java-demo/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Java Sample Project This is an project created to use Maven-published Scala/Java package with two Java examples. diff --git a/scala-package/mxnet-demo/scala-demo/README.md b/scala-package/mxnet-demo/scala-demo/README.md index 8a5abf56894b..6c638194687c 100644 --- a/scala-package/mxnet-demo/scala-demo/README.md +++ b/scala-package/mxnet-demo/scala-demo/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # MXNet Scala Sample Project This is an project created to use Maven-published Scala package with two Scala examples. diff --git a/scala-package/native/README.md b/scala-package/native/README.md index 17e913f1a4eb..4ca6e5f75eaa 100644 --- a/scala-package/native/README.md +++ b/scala-package/native/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Scala JNI diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md index f14cdc09c180..ccbafd46a121 100644 --- a/scala-package/packageTest/README.md +++ b/scala-package/packageTest/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Scala Package Test diff --git a/scala-package/spark/README.md b/scala-package/spark/README.md index 79f637a3dfbf..0b18db0a7cb4 100644 --- a/scala-package/spark/README.md +++ b/scala-package/spark/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + Deep Learning on Spark ===== diff --git a/tests/README.md b/tests/README.md index be09cadff3ca..de5d8107a790 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Testing MXNET diff --git a/tests/nightly/README.md b/tests/nightly/README.md index 5cf41eb2c3ef..774a975edad6 100644 --- a/tests/nightly/README.md +++ b/tests/nightly/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Nightly Tests for MXNet diff --git a/tests/nightly/apache_rat_license_check/README.md b/tests/nightly/apache_rat_license_check/README.md index 0d6d37dc7da8..70ec665fa57f 100644 --- a/tests/nightly/apache_rat_license_check/README.md +++ b/tests/nightly/apache_rat_license_check/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Apache RAT License Check diff --git a/tests/nightly/broken_link_checker_test/README.md b/tests/nightly/broken_link_checker_test/README.md index aaad68601798..d493390c844f 100644 --- a/tests/nightly/broken_link_checker_test/README.md +++ b/tests/nightly/broken_link_checker_test/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Broken link checker test diff --git a/tests/nightly/model_backwards_compatibility_check/README.md b/tests/nightly/model_backwards_compatibility_check/README.md index af17396f0e0f..ae7c7c21015e 100644 --- a/tests/nightly/model_backwards_compatibility_check/README.md +++ b/tests/nightly/model_backwards_compatibility_check/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Model Backwards Compatibility Tests diff --git a/tests/nightly/straight_dope/README.md b/tests/nightly/straight_dope/README.md index 869d80afcdfa..88153aebbc36 100644 --- a/tests/nightly/straight_dope/README.md +++ b/tests/nightly/straight_dope/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Nightly Tests for MXNet: The Straight Dope diff --git a/tests/python/README.md b/tests/python/README.md index fd2282d67b9d..8c09ff03ed1a 100644 --- a/tests/python/README.md +++ b/tests/python/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + Python Test Case ================ diff --git a/tools/accnn/README.md b/tools/accnn/README.md index ca6d735bba39..9c08cb799d6b 100644 --- a/tools/accnn/README.md +++ b/tools/accnn/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Accelerate Convolutional Neural Networks diff --git a/tools/bandwidth/README.md b/tools/bandwidth/README.md index f82e3218a19c..4109a3ce39a8 100644 --- a/tools/bandwidth/README.md +++ b/tools/bandwidth/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Measure communication bandwidth diff --git a/tools/caffe_converter/README.md b/tools/caffe_converter/README.md index b97b6e42ee5c..22164d550e9d 100644 --- a/tools/caffe_converter/README.md +++ b/tools/caffe_converter/README.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + # Convert Caffe Model to Mxnet Format diff --git a/tools/caffe_translator/README.md b/tools/caffe_translator/README.md index 5d80caea288e..7ec7b7275dd3 100644 --- a/tools/caffe_translator/README.md +++ b/tools/caffe_translator/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Caffe Translator Caffe Translator is a migration tool that helps developers migrate their existing Caffe code to MXNet and continue further development using MXNet. Note that this is different from the Caffe to MXNet model converter which is available [here](https://github.com/apache/incubator-mxnet/tree/master/tools/caffe_converter). diff --git a/tools/caffe_translator/build_from_source.md b/tools/caffe_translator/build_from_source.md index f51d2fcaf3f0..c08a423a44e7 100644 --- a/tools/caffe_translator/build_from_source.md +++ b/tools/caffe_translator/build_from_source.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + ### Build Caffe Translator from source diff --git a/tools/caffe_translator/faq.md b/tools/caffe_translator/faq.md index 186c0f623a14..e187257e66a9 100644 --- a/tools/caffe_translator/faq.md +++ b/tools/caffe_translator/faq.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + ### Frequently asked questions diff --git a/tools/cfn/Readme.md b/tools/cfn/Readme.md index ecbdf836c9ef..16e3cf5802c4 100644 --- a/tools/cfn/Readme.md +++ b/tools/cfn/Readme.md @@ -1,21 +1,19 @@ - + + + + + + - http://www.apache.org/licenses/LICENSE-2.0 + - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> + + + + + + **Distributed Deep Learning Made Easy has found more love and new home, please visit [awslabs/deeplearning-cfn](https://github.com/awslabs/deeplearning-cfn)** \ No newline at end of file diff --git a/tools/coreml/README.md b/tools/coreml/README.md index 87f0a953dc71..31982babea78 100644 --- a/tools/coreml/README.md +++ b/tools/coreml/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Convert MXNet models into Apple CoreML format. diff --git a/tools/dependencies/README.md b/tools/dependencies/README.md index c8a06d868b44..1c2f5172a783 100644 --- a/tools/dependencies/README.md +++ b/tools/dependencies/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # Overview diff --git a/tools/staticbuild/README.md b/tools/staticbuild/README.md index 3297bbdfbd40..bfccbab184bd 100644 --- a/tools/staticbuild/README.md +++ b/tools/staticbuild/README.md @@ -1,21 +1,19 @@ - + + + + + + + + + + + + + + + + # MXNet Static Build From e35658628617dbf1a078805f767002e7e589c282 Mon Sep 17 00:00:00 2001 From: Carin Meier Date: Wed, 13 Feb 2019 22:34:37 -0500 Subject: [PATCH 16/75] disable flaky integration test (#14151) --- contrib/clojure-package/integration-tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/clojure-package/integration-tests.sh b/contrib/clojure-package/integration-tests.sh index ce480a507f92..3f80ea5fb29a 100755 --- a/contrib/clojure-package/integration-tests.sh +++ b/contrib/clojure-package/integration-tests.sh @@ -26,7 +26,7 @@ lein install # then run through the examples EXAMPLES_HOME=${MXNET_HOME}/contrib/clojure-package/examples # use AWK pattern for blacklisting -TEST_CASES=`find ${EXAMPLES_HOME} -name test | awk '!/dontselect1|dontselect2/'` +TEST_CASES=`find ${EXAMPLES_HOME} -name test | awk '!/dontselect1|cnn-text-classification/'` for i in $TEST_CASES ; do cd ${i} && lein test -done \ No newline at end of file +done From 074a859a00dfbcfae17e88b5d2c1b5d7dec71c74 Mon Sep 17 00:00:00 2001 From: perdasilva Date: Thu, 14 Feb 2019 18:38:16 +0100 Subject: [PATCH 17/75] Fixes installation nightly test by filtering out the git commands (#14144) --- tests/jenkins/run_test_installation_docs.sh | 31 ++++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/tests/jenkins/run_test_installation_docs.sh b/tests/jenkins/run_test_installation_docs.sh index d25f1d500c8f..4f62865adac0 100755 --- a/tests/jenkins/run_test_installation_docs.sh +++ b/tests/jenkins/run_test_installation_docs.sh @@ -250,6 +250,29 @@ function set_instruction_set() { ${sorted_indexes[$end_buildfromsource_command_index]}) } +# given a $buildfromsource_commands string, filter out any build commands that should not be executed +# during the build from source tests. An example, the build from source instructions include the commands: +# $ git clone --recursive https://github.com/apache/incubator-mxnet +# $ cd incubator-mxnet +# if these commands get executed in the jenkins job, we will be testing the build from source instructions +# against the master branch and not against the version of the repository that Jenkins checksout for testing. +# This presents a particularly big problem for the version branches and their nightly builds. Because, +# we would, in effect, be testing the build from source instructions for one version of MXNet against +# the master branch. +# in this function we target the commands cited in the example above, but leave it open for expantion +# in the future. +# See also gh issue: https://github.com/apache/incubator-mxnet/issues/13800 +function filter_build_commands() { + filtered_build_commands="${1}" + + # Remove git commands + filtered_build_commands=`echo "${filtered_build_commands}" | perl -pe 's/git .*?;//g'` + + # Remove 'cd incubator-mxnet' + filtered_build_commands=`echo "${filtered_build_commands}" | perl -pe 's/cd incubator-mxnet;//'` + + echo "${filtered_build_commands}" +} ########################LINUX-PYTHON-CPU############################ echo @@ -302,8 +325,8 @@ ubuntu_python_cpu_source() set -e echo echo "### Testing Build From Source ###" - echo "${buildfromsource_commands}" - echo + buildfromsource_commands=$(filter_build_commands "${buildfromsource_commands}") + echo ${buildfromsource_commands} eval ${buildfromsource_commands} echo "ubuntu_python_cpu_source: MXNet Installed Successfully" @@ -363,8 +386,8 @@ ubuntu_python_gpu_source() set -e echo echo "### Testing Build From Source ###" - echo "${buildfromsource_commands}" - echo + buildfromsource_commands=$(filter_build_commands "${buildfromsource_commands}") + echo ${buildfromsource_commands} eval ${buildfromsource_commands} echo "ubuntu_python_gpu_source: MXNet Installed Successfully" From 518cd408a2c1d82dd5342268dfc80c41e427b765 Mon Sep 17 00:00:00 2001 From: Amol Lele <19983848+leleamol@users.noreply.github.com> Date: Thu, 14 Feb 2019 09:55:09 -0800 Subject: [PATCH 18/75] Updated the MLP test to accept the number of epochs. Reduced the epochs in ci_test.sh to shorten the CI build time (#14149) --- cpp-package/example/mlp.cpp | 7 +++---- cpp-package/tests/ci_test.sh | 10 +++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cpp-package/example/mlp.cpp b/cpp-package/example/mlp.cpp index cc16f53cf205..c3760fd3c846 100644 --- a/cpp-package/example/mlp.cpp +++ b/cpp-package/example/mlp.cpp @@ -49,7 +49,7 @@ void OutputAccuracy(mx_float* pred, mx_float* target) { std::cout << "Accuracy: " << right / 128.0 << std::endl; } -void MLP() { +void MLP(int max_epoch) { auto sym_x = Symbol::Variable("X"); auto sym_label = Symbol::Variable("label"); @@ -144,7 +144,6 @@ void MLP() { grad_req_type, aux_states); std::cout << "Training" << std::endl; - int max_epoch = 15000; mx_float learning_rate = 0.0001; for (int epoch_num = 0; epoch_num < max_epoch; ++epoch_num) { exe->Forward(true); @@ -173,8 +172,8 @@ void MLP() { } int main(int argc, char** argv) { - MLP(); + int max_epoch = argc > 1 ? strtol(argv[1], NULL, 10) : 15000; + MLP(max_epoch); MXNotifyShutdown(); return 0; } - diff --git a/cpp-package/tests/ci_test.sh b/cpp-package/tests/ci_test.sh index 7abdef481ec9..18fabea7a7f9 100755 --- a/cpp-package/tests/ci_test.sh +++ b/cpp-package/tests/ci_test.sh @@ -25,22 +25,22 @@ ls -l ../../lib/ ./get_data.sh cp ../../build/cpp-package/example/lenet . -./lenet 10 +./lenet 1 cp ../../build/cpp-package/example/alexnet . ./alexnet 1 cp ../../build/cpp-package/example/lenet_with_mxdataiter . -./lenet_with_mxdataiter 5 +./lenet_with_mxdataiter 1 cp ../../build/cpp-package/example/resnet . -./resnet 5 +./resnet 1 cp ../../build/cpp-package/example/inception_bn . -./inception_bn 5 +./inception_bn 1 cp ../../build/cpp-package/example/mlp . -./mlp +./mlp 150 cp ../../build/cpp-package/example/mlp_cpu . ./mlp_cpu From a13f8d92fde7e377e05c19a1b7cd4b66f0e0c28a Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 14 Feb 2019 19:26:59 +0100 Subject: [PATCH 19/75] Add license check to dev_menu, docs build with docker (#14166) --- dev_menu.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dev_menu.py b/dev_menu.py index 7329b446ed34..1fda4c7aabca 100755 --- a/dev_menu.py +++ b/dev_menu.py @@ -118,10 +118,13 @@ def create_virtualenv_default(): ('[Local] Python Unit tests', "./py3_venv/bin/nosetests -v tests/python/unittest/" ), - ('[Website and docs build] Will build to docs/_build/html/', - "ci/docker/runtime_functions.sh deploy_docs"), - ('[Docker] sanity_check. Check for linting and code formatting.', - "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh sanity_check"), + ('[Docker] Website and docs build outputs to "docs/_build/html/"', + "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh deploy_docs"), + ('[Docker] sanity_check. Check for linting and code formatting and licenses.', + [ + "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh sanity_check", + "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh nightly_test_rat_check", + ]), ('[Docker] Python3 CPU unittests', [ "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_openblas", From 73a9f1ceb3d96b819472050514a59a5eae4baa92 Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Thu, 14 Feb 2019 11:15:10 -0800 Subject: [PATCH 20/75] Relaxing type requirements for slice_like op (#14097) * Relaxing types for slice_like op * Added test * Fix typo in test * Fix lint --- src/operator/tensor/matrix_op.cc | 11 ++++++++++- tests/python/unittest/test_operator.py | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/operator/tensor/matrix_op.cc b/src/operator/tensor/matrix_op.cc index e5d354be629f..3a244ac89790 100644 --- a/src/operator/tensor/matrix_op.cc +++ b/src/operator/tensor/matrix_op.cc @@ -661,7 +661,16 @@ Example:: return std::vector{"data", "shape_like"}; }) .set_attr("FInferShape", SliceLikeShape) -.set_attr("FInferType", ElemwiseType<2, 1>) +.set_attr("FInferType", [](const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), 2) << " in operator " << attrs.name; + std::vector checked_in_attrs = { (*in_attrs)[0] }; + bool ret = !type_is_none((*in_attrs)[1]) && + ElemwiseType<1, 1>(attrs, &checked_in_attrs, out_attrs); + (*in_attrs)[0] = checked_in_attrs[0]; + return ret; + }) .set_attr("FGradient", ElemwiseGradUseNone{"_backward_slice_like"}) .set_attr("FCompute", SliceLikeForward) .add_argument("data", "NDArray-or-Symbol", "Source input") diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 1f422155f512..fc003b2271ef 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -2515,6 +2515,20 @@ def test_slice_like(): assert_allclose(xx, xgrad.asnumpy()) assert_allclose(xgrad1.asnumpy(), mx.nd.zeros_like(xgrad1).asnumpy()) +@with_seed() +def test_slice_like_different_types(): + x = [[ 1., 2., 3., 4.], + [ 5., 6., 7., 8.], + [ 9., 10., 11., 12.]] + + y = [[ 0., 0., 0.], + [ 0., 0., 0.]] + + x = mx.nd.array(x) + y = mx.nd.array(y).astype('int32') + z = mx.nd.slice_like(x, y) + assert_allclose(z.asnumpy(), [[1,2,3],[5,6,7]]) + @with_seed() def test_flip(): for ndim in range(1, 6): From dad33b51b4a3a05f9bb94f781628494a409e80e4 Mon Sep 17 00:00:00 2001 From: Sandeep Krishnamurthy Date: Thu, 14 Feb 2019 11:55:43 -0800 Subject: [PATCH 21/75] Performance improvement in Normalize GPU Kernel (#14139) * New CPU kernel for normalize * New GPU kernel for Normalize * Add launch bounds and increase threads to 32*32 * do not hardcode number of threads * Try fix windows build failure * make channels as int to fix windows build issues with omp * Simplify cuda kernels with 1 D thread block * Minor refactoring * Revert thread dim for ToTensor operator --- src/operator/image/image_random-inl.h | 307 ++++++++++++++++---------- src/operator/image/image_random.cu | 170 ++++++++++++-- 2 files changed, 336 insertions(+), 141 deletions(-) diff --git a/src/operator/image/image_random-inl.h b/src/operator/image/image_random-inl.h index 392fff4dbf81..0f4d173be79a 100644 --- a/src/operator/image/image_random-inl.h +++ b/src/operator/image/image_random-inl.h @@ -54,6 +54,35 @@ void ToTensorImplCUDA(mshadow::Stream *s, const T2 output, const int req, const float normalize_factor); + +template +void NormalizeImplCUDA(mshadow::Stream *s, + const DType *input, + DType *output, + const int req, + const int N, + const int C, + const int H, + const int W, + const float mean_d0, + const float mean_d1, + const float mean_d2, + const float std_d0, + const float std_d1, + const float std_d2); + +template +void NormalizeBackwardImplCUDA(mshadow::Stream *s, + const DType *out_grad, + DType *in_grad, + const int req, + const int N, + const int C, + const int H, + const int W, + const float std_d0, + const float std_d1, + const float std_d2); #endif // MXNET_USE_CUDA // Shape and Type inference for image to tensor operator @@ -254,156 +283,165 @@ inline bool NormalizeOpType(const nnvm::NodeAttrs& attrs, return out_attrs->at(0) != -1; } -template -struct normalize_forward { - template - MSHADOW_XINLINE static void Map(uint32_t c, DType* out_data, const DType* in_data, - const float mean_d0, const float mean_d1, const float mean_d2, - const float std_d0, const float std_d1, const float std_d2, - const int length, const int step) { - float mean, std; - switch (c) { - case 0 : mean = mean_d0; - std = std_d0; - break; - case 1 : mean = mean_d1; - std = std_d1; - break; - case 2 : mean = mean_d2; - std = std_d2; - break; - } - #pragma omp parallel for - for (int i = 0; i < length; ++i) { - KERNEL_ASSIGN(out_data[step + c*length + i], req, - (in_data[step + c*length + i] - mean) / std); - } +template +inline void Normalize(DType* out_data, + const DType* in_data, + const int length, + const int channels, + const int step, + const std::vector mean, + const std::vector std) { + // Microsoft Visual C++ compiler does not support omp collapse + #ifdef _MSC_VER + #pragma omp parallel for + #else + #pragma omp parallel for collapse(2) + #endif // _MSC_VER + for (int c = 0; c < channels; ++c) { + for (int i = 0; i < length; ++i) { + KERNEL_ASSIGN(out_data[step + c*length + i], req, + (in_data[step + c*length + i] - mean[c]) / std[c]); } -}; - -template -void NormalizeImpl(const OpContext &ctx, - const std::vector &inputs, - const std::vector &outputs, - const std::vector &req, - const float mean_d0, const float mean_d1, - const float mean_d2, const float std_d0, - const float std_d1, const float std_d2, - const int length, - const uint32_t channel, - const int step = 0) { - mshadow::Stream *s = ctx.get_stream(); + } +} - MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, { - MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, { - DType* input = inputs[0].dptr(); - DType* output = outputs[0].dptr(); - mxnet_op::Kernel, xpu>::Launch( - s, channel, output, input, mean_d0, mean_d1, mean_d2, - std_d0, std_d1, std_d2, length, step); - }); +inline void NormalizeImpl(const std::vector &inputs, + const std::vector &outputs, + const std::vector &req, + const int length, + const int channels, + const int step, + const std::vector mean, + const std::vector std) { + MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, { + MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, { + DType* input = inputs[0].dptr(); + DType* output = outputs[0].dptr(); + Normalize(output, input, length, channels, step, + mean, std); }); + }); } template void NormalizeOpForward(const nnvm::NodeAttrs &attrs, - const OpContext &ctx, - const std::vector &inputs, - const std::vector &req, - const std::vector &outputs) { + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs) { CHECK_EQ(inputs.size(), 1U); CHECK_EQ(outputs.size(), 1U); CHECK_EQ(req.size(), 1U); const NormalizeParam ¶m = nnvm::get(attrs.parsed); - // Note: We need mean and std_dev in the kernel. - // It is costly (device copy) to pass it as vector, for gpu kernel. - // Hence, passing it as below for performance. - float mean_d0, mean_d1, mean_d2; - float std_d0, std_d1, std_d2; - - // Mean and Std can be 1 or 3 D only. + // Mean and Std can be 1 or 3D only. + std::vector mean(3); + std::vector std(3); if (param.mean.ndim() == 1) { - mean_d0 = mean_d1 = mean_d2 = param.mean[0]; + mean[0] = mean[1] = mean[3] = param.mean[0]; } else { - mean_d0 = param.mean[0]; - mean_d1 = param.mean[1]; - mean_d2 = param.mean[2]; + mean[0] = param.mean[0]; + mean[1] = param.mean[1]; + mean[2] = param.mean[2]; } if (param.std.ndim() == 1) { - std_d0 = std_d1 = std_d2 = param.std[0]; + std[0] = std[1] = std[2] = param.std[0]; } else { - std_d0 = param.std[0]; - std_d1 = param.std[1]; - std_d2 = param.std[2]; + std[0] = param.std[0]; + std[1] = param.std[1]; + std[2] = param.std[2]; } - // 3D input (c, h, w) - if (inputs[0].ndim() == 3) { + if (std::is_same::value) { + #if MXNET_USE_CUDA + mshadow::Stream *s = ctx.get_stream(); + MSHADOW_TYPE_SWITCH(inputs[0].type_flag_, DType, { + MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, { + int N, C, H, W; + DType *input = nullptr; + DType *output = nullptr; + if (inputs[0].ndim() == 3) { + N = 1; + C = static_cast(inputs[0].shape_[0]); + H = static_cast(inputs[0].shape_[1]); + W = static_cast(inputs[0].shape_[2]); + input = (inputs[0].get(s)).dptr_; + output = (outputs[0].get(s)).dptr_; + } else { + N = static_cast(inputs[0].shape_[0]); + C = static_cast(inputs[0].shape_[1]); + H = static_cast(inputs[0].shape_[2]); + W = static_cast(inputs[0].shape_[3]); + input = (inputs[0].get(s)).dptr_; + output = (outputs[0].get(s)).dptr_; + } + NormalizeImplCUDA(s, input, output, req_type, + N, C, H, W, + mean[0], mean[1], mean[2], + std[0], std[1], std[2]); + }); + }); + #else + LOG(FATAL) << "Compile with USE_CUDA=1 to use Normalize operator on GPU."; + #endif // MXNET_USE_CUDA + } else if (inputs[0].ndim() == 3) { + // 3D input (c, h, w) const int length = inputs[0].shape_[1] * inputs[0].shape_[2]; - const uint32_t channel = inputs[0].shape_[0]; - NormalizeImpl(ctx, inputs, outputs, req, mean_d0, mean_d1, mean_d2, - std_d0, std_d1, std_d2, length, channel); + const int channel = static_cast(inputs[0].shape_[0]); + const int step = 0; + NormalizeImpl(inputs, outputs, req, length, channel, step, mean, std); } else if (inputs[0].ndim() == 4) { // 4D input (n, c, h, w) const int batch_size = inputs[0].shape_[0]; const int length = inputs[0].shape_[2] * inputs[0].shape_[3]; - const uint32_t channel = inputs[0].shape_[1]; + const int channel = static_cast(inputs[0].shape_[1]); const int step = channel * length; #pragma omp parallel for for (auto n = 0; n < batch_size; ++n) { - NormalizeImpl(ctx, inputs, outputs, req, mean_d0, mean_d1, mean_d2, - std_d0, std_d1, std_d2, length, channel, n*step); + NormalizeImpl(inputs, outputs, req, length, channel, n*step, mean, std); } } } // Backward function -template -struct normalize_backward { - template - MSHADOW_XINLINE static void Map(uint32_t c, DType* in_grad, const DType* out_grad, - const float std_d0, const float std_d1, const float std_d2, - const int length, const int step) { - // d/dx{(x - mean) / std_dev} => (1 / std_dev) - float std_dev; - switch (c) { - case 0 : std_dev = std_d0; - break; - case 1 : std_dev = std_d1; - break; - case 2 : std_dev = std_d2; - break; - } - +template +inline void NormalizeBackward(const DType* out_grad, + DType* in_grad, + const int length, + const int channels, + const int step, + const std::vector std) { + // Microsoft Visual C++ compiler does not support omp collapse + #ifdef _MSC_VER #pragma omp parallel for + #else + #pragma omp parallel for collapse(2) + #endif // _MSC_VER + for (int c = 0; c < channels; ++c) { for (int i = 0; i < length; ++i) { KERNEL_ASSIGN(in_grad[step + c*length + i], req, - out_grad[step + c*length + i] * (1.0 / std_dev)); + out_grad[step + c*length + i] * (1.0 / std[c])); } } -}; - -template -void NormalizeBackwardImpl(const OpContext &ctx, - const std::vector &inputs, - const std::vector &outputs, - const std::vector &req, - const float std_d0, const float std_d1, const float std_d2, - const int length, - const uint32_t channel, - const int step = 0) { - mshadow::Stream *s = ctx.get_stream(); +} +inline void NormalizeBackwardImpl(const std::vector &inputs, + const std::vector &outputs, + const std::vector &req, + const int length, + const int channels, + const int step, + const std::vector std + ) { MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, { MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, { DType* out_grad = inputs[0].dptr(); DType* in_grad = outputs[0].dptr(); - mxnet_op::Kernel, xpu>::Launch( - s, channel, in_grad, out_grad, std_d0, std_d1, std_d2, length, step); + NormalizeBackward(out_grad, in_grad, length, + channels, step, std); }); }); } @@ -419,37 +457,66 @@ void NormalizeOpBackward(const nnvm::NodeAttrs &attrs, CHECK_EQ(req.size(), 1U); const NormalizeParam ¶m = nnvm::get(attrs.parsed); - float std_d0, std_d1, std_d2; - - // Std can be 1 or 3 D only + // Std can be 1 or 3D only. + std::vector std(3); if (param.std.ndim() == 1) { - std_d0 = std_d1 = std_d2 = param.std[0]; + std[0] = std[1] = std[2] = param.std[0]; } else { - std_d0 = param.std[0]; - std_d1 = param.std[1]; - std_d2 = param.std[2]; + std[0] = param.std[0]; + std[1] = param.std[1]; + std[2] = param.std[2]; } // Note: inputs[0] is out_grad const TBlob& in_data = inputs[1]; - // 3D input (c, h, w) - if (in_data.ndim() == 3) { + if (std::is_same::value) { + #if MXNET_USE_CUDA + mshadow::Stream *s = ctx.get_stream(); + MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, { + MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, { + int N, C, H, W; + DType *in_grad = nullptr; + DType *out_grad = nullptr; + if (in_data.ndim() == 3) { + N = 1; + C = static_cast(in_data.shape_[0]); + H = static_cast(in_data.shape_[1]); + W = static_cast(in_data.shape_[2]); + out_grad = (inputs[0].get(s)).dptr_; + in_grad = (outputs[0].get(s)).dptr_; + } else { + N = static_cast(in_data.shape_[0]); + C = static_cast(in_data.shape_[1]); + H = static_cast(in_data.shape_[2]); + W = static_cast(in_data.shape_[3]); + out_grad = (inputs[0].get(s)).dptr_; + in_grad = (outputs[0].get(s)).dptr_; + } + NormalizeBackwardImplCUDA(s, out_grad, in_grad, req_type, + N, C, H, W, + std[0], std[1], std[2]); + }); + }); + #else + LOG(FATAL) << "Compile with USE_CUDA=1 to use Normalize backward operator on GPU."; + #endif // MXNET_USE_CUDA + } else if (in_data.ndim() == 3) { + // 3D input (c, h, w) const int length = in_data.shape_[1] * in_data.shape_[2]; - const uint32_t channel = in_data.shape_[0]; - NormalizeBackwardImpl(ctx, inputs, outputs, req, std_d0, std_d1, std_d2, length, channel); + const int channel = static_cast(in_data.shape_[0]); + const int step = 0; + NormalizeBackwardImpl(inputs, outputs, req, length, channel, step, std); } else if (in_data.ndim() == 4) { // 4D input (n, c, h, w) const int batch_size = in_data.shape_[0]; const int length = in_data.shape_[2] * in_data.shape_[3]; - const uint32_t channel = in_data.shape_[1]; + const int channel = static_cast(in_data.shape_[1]); const int step = channel * length; #pragma omp parallel for for (auto n = 0; n < batch_size; ++n) { - NormalizeBackwardImpl(ctx, inputs, outputs, req, - std_d0, std_d1, std_d2, length, - channel, n*step); + NormalizeBackwardImpl(inputs, outputs, req, length, channel, n*step, std); } } } diff --git a/src/operator/image/image_random.cu b/src/operator/image/image_random.cu index 6fe53832a89e..5dfbbd3b6b00 100644 --- a/src/operator/image/image_random.cu +++ b/src/operator/image/image_random.cu @@ -32,15 +32,25 @@ namespace image { using namespace mshadow; // ToTensor Kernel for 3D input +/* + * In order to not generate the code that uses too many + * registers (resulting in too many resources requested + * error) we need to tell the compiler that we will be + * launching this kernel with cuda::kMaxThreadsPerBlock + * threads per block. Setting __launch_bounds__ ensures + * that such configuration can always be launched. + */ template -__global__ void ToTensorCudaKernel(const Tensor input, - const Tensor output, - const int req, - const int N, - const int H, - const int W, - const int C, - const float normalize_factor) { +__global__ void +__launch_bounds__(cuda::kMaxThreadsPerBlock, 1) +ToTensorCudaKernel(const Tensor input, + const Tensor output, + const int req, + const int N, + const int H, + const int W, + const int C, + const float normalize_factor) { // We process one image per thread block. // In 3D case, we have only 1 block i.e., blockIdx.x // We do not use it. @@ -56,14 +66,16 @@ __global__ void ToTensorCudaKernel(const Tensor input, // ToTensor Kernel for 4D input template -__global__ void ToTensorCudaKernel(const Tensor input, - const Tensor output, - const int req, - const int N, - const int H, - const int W, - const int C, - const float normalize_factor) { +__global__ void +__launch_bounds__(cuda::kMaxThreadsPerBlock, 1) +ToTensorCudaKernel(const Tensor input, + const Tensor output, + const int req, + const int N, + const int H, + const int W, + const int C, + const float normalize_factor) { // We process one image per thread block. const int n = blockIdx.x; @@ -99,18 +111,134 @@ void ToTensorImplCUDA(mshadow::Stream *s, W = input.size(2); C = input.size(3); blocks = N > 0 ? N : 1; - blocks = N; } - // One block per image. - // Number of threads = (32, 32) is optimal, because, - // computation is minimal and overhead of CUDA preparing - // all threads is minimal. + ToTensorCudaKernel <<>>(input, output, req, N, H, W, C, normalize_factor); MSHADOW_CUDA_POST_KERNEL_CHECK(ToTensorCudaKernel); } +// Normalize Forward CUDA Kernel +template +__global__ void +__launch_bounds__(cuda::kMaxThreadsPerBlock, 1) +NormalizeCudaKernel(const DType* input, + DType* output, + const int req, + const int N, + const int C, + const int H, + const int W, + const float mean_d0, + const float mean_d1, + const float mean_d2, + const float std_d0, + const float std_d1, + const float std_d2) { + // We process one image per thread block. + const int n = blockIdx.x; + const int length = H * W; + const int step = C * length * n; + + float mean = mean_d0; + float std = std_d0; + for (int c = 0; c < C; ++c) { + switch (c) { + case 0 : break; + case 1 : mean = mean_d1; + std = std_d1; + break; + case 2 : mean = mean_d2; + std = std_d2; + break; + } + for (int i = threadIdx.x; i < length; i += blockDim.x) { + KERNEL_ASSIGN(*(output + step + i + (c * length)), req, + (*(input + step + i + (c * length)) - mean) / std); + } + } +} + +template +void NormalizeImplCUDA(mshadow::Stream *s, + const DType* input, + DType* output, + const int req, + const int N, + const int C, + const int H, + const int W, + const float mean_d0, + const float mean_d1, + const float mean_d2, + const float std_d0, + const float std_d1, + const float std_d2) { + cudaStream_t stream = mshadow::Stream::GetStream(s); + NormalizeCudaKernel + // 1 image per block. N is batch size. + <<>>(input, output, + req, N, C, H, W, mean_d0, mean_d1, mean_d2, + std_d0, std_d1, std_d2); + MSHADOW_CUDA_POST_KERNEL_CHECK(NormalizeCudaKernel); +} + +// Normalize Backward Kernel +template +__global__ void +__launch_bounds__(cuda::kMaxThreadsPerBlock, 1) +NormalizeBackwardCudaKernel(const DType *out_grad, + DType *in_grad, + const int req, + const int N, + const int C, + const int H, + const int W, + const float std_d0, + const float std_d1, + const float std_d2) { + // We process one image per thread block. + const int n = blockIdx.x; + const int length = H * W; + const int step = C * length * n; + + float std = std_d0; + for (int c = 0; c < C; ++c) { + switch (c) { + case 0 : break; + case 1 : std = std_d1; + break; + case 2 : std = std_d2; + break; + } + for (int i = threadIdx.x; i < length; i += blockDim.x) { + KERNEL_ASSIGN(*(in_grad + step + i + (c * length)), req, + *(out_grad + step + i + (c * length)) * (1.0 / std)); + } + } +} + +template +void NormalizeBackwardImplCUDA(mshadow::Stream *s, + const DType *out_grad, + DType *in_grad, + const int req, + const int N, + const int C, + const int H, + const int W, + const float std_d0, + const float std_d1, + const float std_d2) { + cudaStream_t stream = mshadow::Stream::GetStream(s); + NormalizeBackwardCudaKernel + // 1 image per block. N is batch size. + <<>>(out_grad, in_grad, + req, N, C, H, W, std_d0, std_d1, std_d2); + MSHADOW_CUDA_POST_KERNEL_CHECK(NormalizeBackwardCudaKernel); +} + NNVM_REGISTER_OP(_image_to_tensor) .set_attr("FCompute", ToTensorOpForward); From df5310b059417fb6d855f99cf7bb0cf7c0093551 Mon Sep 17 00:00:00 2001 From: Junru Shao Date: Thu, 14 Feb 2019 13:19:53 -0800 Subject: [PATCH 22/75] [MXNET-1315] Add checks for dynamic-shaped operators in CachedOp (#14018) * Initial commit * Try this * Boy next door! * Add comments per discussion with Da * Try this * boy try this * change the boss of the gym --- src/imperative/cached_op.cc | 10 +++++++++- src/imperative/cached_op.h | 4 ++++ src/imperative/imperative_utils.h | 15 +++++++++++---- src/operator/contrib/boolean_mask.cc | 2 ++ src/operator/subgraph_op_common.cc | 8 +++++++- src/operator/subgraph_op_common.h | 3 ++- 6 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 58ec4e65b846..8dd0a4deaac3 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -285,12 +285,20 @@ bool CachedOp::SetForwardGraph( } bool match = true; - match &= CheckAndInferShape(&g, std::move(shape_inputs), true); + bool contain_dynamic_shape = false; + match &= CheckAndInferShape(&g, std::move(shape_inputs), true, + {0, 0}, {0, 0}, &contain_dynamic_shape); match &= CheckAndInferType(&g, std::move(dtype_inputs), true); exec::DevMaskVector dev_mask(g.indexed_graph().num_nodes(), inputs[0]->ctx().dev_mask()); match &= CheckAndInferStorageType(&g, std::move(dev_mask), std::move(storage_type_inputs), true); + // When dynmaic shape exists, it is not feasible to plan memory ahead of time + if (contain_dynamic_shape) { + g.attrs.erase("forward_mem_plan"); + g.attrs.erase("full_mem_plan"); + return false; + } if (!match) { g.attrs.erase("forward_mem_plan"); g.attrs.erase("full_mem_plan"); diff --git a/src/imperative/cached_op.h b/src/imperative/cached_op.h index 59a793ee1b65..3b173c8654a4 100644 --- a/src/imperative/cached_op.h +++ b/src/imperative/cached_op.h @@ -35,6 +35,7 @@ struct CachedOpConfig : public dmlc::Parameter { uint32_t backward_bulk_size; bool static_alloc; bool static_shape; + bool is_dynamic; nnvm::Tuple data_indices; nnvm::Tuple param_indices; std::string subgraph; @@ -66,6 +67,9 @@ struct CachedOpConfig : public dmlc::Parameter { DMLC_DECLARE_FIELD(subgraph) .set_default(std::string("")) .describe("JSON string of a subgraph."); + DMLC_DECLARE_FIELD(is_dynamic) + .set_default(false) + .describe("Whether the graph contains dynamic shape operators."); } }; diff --git a/src/imperative/imperative_utils.h b/src/imperative/imperative_utils.h index 6446c37627a9..7113cb2063c1 100644 --- a/src/imperative/imperative_utils.h +++ b/src/imperative/imperative_utils.h @@ -179,7 +179,7 @@ inline void SetShapeType(const Context& ctx, for (size_t i = 0; i < outputs.size(); ++i) { NDArrayStorageType storage_type = static_cast(out_storage_types[i]); - if (outputs[i]->is_none()) { + if (outputs[i]->is_none() || outputs[i]->shape().ndim() == 0) { if (is_dynamic_shape_existing) { // once there is dynamic shape somewhere, we could not pre-determine the shape. *outputs[i] = NDArray(ctx, out_types[i]); @@ -566,8 +566,12 @@ inline void PushOperator(const OpStatePtr& state, inline bool CheckAndInferShape(nnvm::Graph* p_g, nnvm::ShapeVector&& shapes, bool use_inputs, std::pair node_range = {0, 0}, - std::pair entry_range = {0, 0}) { + std::pair entry_range = {0, 0}, + bool *contain_unknown = nullptr) { using namespace nnvm; + if (contain_unknown != nullptr) { + *contain_unknown = false; + } nnvm::Graph& g = *p_g; if (use_inputs) { if (g.attrs.count("shape_inputs") && @@ -601,8 +605,11 @@ inline bool CheckAndInferShape(nnvm::Graph* p_g, nnvm::ShapeVector&& shapes, g.attrs["shape"] = std::make_shared(std::move(shapes)); g = exec::InferShape(std::move(g)); } - CHECK_EQ(g.GetAttr("shape_num_unknown_nodes"), 0U); - + if (contain_unknown == nullptr) { + CHECK_EQ(g.GetAttr("shape_num_unknown_nodes"), 0U); + } else { + *contain_unknown = g.GetAttr("shape_num_unknown_nodes") != 0U; + } return false; } diff --git a/src/operator/contrib/boolean_mask.cc b/src/operator/contrib/boolean_mask.cc index 2dcafb6b9494..7fd66bc321bf 100644 --- a/src/operator/contrib/boolean_mask.cc +++ b/src/operator/contrib/boolean_mask.cc @@ -98,6 +98,8 @@ which stands for the rows in x where the corresonding element in index is non-ze .set_attr("FComputeEx", BooleanMaskForward) .set_attr("FInferStorageType", BooleanMaskStorageType) .set_attr("FGradient", ElemwiseGradUseIn{"_backward_contrib_boolean_mask"}) +.set_attr("FListInputNames", [](const NodeAttrs& attrs) { + return std::vector{"data", "index"};}) .add_argument("data", "NDArray-or-Symbol", "Data") .add_argument("index", "NDArray-or-Symbol", "Mask") .add_arguments(BooleanMaskParam::__FIELDS__()); diff --git a/src/operator/subgraph_op_common.cc b/src/operator/subgraph_op_common.cc index 7a99aedb8602..4b8f63abd4ce 100644 --- a/src/operator/subgraph_op_common.cc +++ b/src/operator/subgraph_op_common.cc @@ -222,8 +222,14 @@ void LoopState::Forward(int iter_no, // If an input and an output share the array, the output array will be changed // by CachedOp. We need to copy data to the real output. for (size_t i = 0; i < out_bufs.size(); i++) - if (!out_bufs[i].IsSame(coutputs[i])) + if (!out_bufs[i].IsSame(coutputs[i])) { + // The line below checks whether dynamic shape exists. + // If so, re-initialize the shape. + if (coutputs[i].shape().ndim() == 0) { + const_cast(coutputs[i]).Init(out_bufs[i].shape()); + } CopyFromTo(out_bufs[i], coutputs[i]); + } if (is_recording) { all_inputs.push_back(cinputs); all_outputs.push_back(coutputs); diff --git a/src/operator/subgraph_op_common.h b/src/operator/subgraph_op_common.h index ebf727f0f5a8..c316fca91d95 100644 --- a/src/operator/subgraph_op_common.h +++ b/src/operator/subgraph_op_common.h @@ -161,7 +161,8 @@ class LoopState { // only static_alloc supports nested call of CachedOp. std::vector > kwargs = { {"inline_limit", "0"}, - {"static_alloc", "1"} + {"static_alloc", "1"}, + {"is_dynamic", "1"} }; return std::make_shared(sym, kwargs); } From de5cda27145d8f5fafda06446d060dcbbb3d3872 Mon Sep 17 00:00:00 2001 From: Holger Kohr Date: Fri, 15 Feb 2019 00:41:52 +0100 Subject: [PATCH 23/75] Add pixelshuffle layers (#13571) * Add pixelshuffle layers, closes #13548 * Remove fmt comments * Use explicit class in super() * Add axis swapping to pixel shuffling, add tests * Add documentation to pixel shuffle layers * Use pixelshuffle layer and fix download in superres example * Add pixelshuffle layers to API doc page --- docs/api/python/gluon/contrib.md | 3 + .../super_resolution/super_resolution.py | 144 +++++++++----- python/mxnet/gluon/contrib/nn/__init__.py | 2 +- python/mxnet/gluon/contrib/nn/basic_layers.py | 181 +++++++++++++++++- tests/python/unittest/test_gluon_contrib.py | 87 ++++++++- 5 files changed, 362 insertions(+), 55 deletions(-) diff --git a/docs/api/python/gluon/contrib.md b/docs/api/python/gluon/contrib.md index b893d5841254..790f6b496516 100644 --- a/docs/api/python/gluon/contrib.md +++ b/docs/api/python/gluon/contrib.md @@ -54,6 +54,9 @@ In the rest of this document, we list routines provided by the `gluon.contrib` p Identity SparseEmbedding SyncBatchNorm + PixelShuffle1D + PixelShuffle2D + PixelShuffle3D ``` ### Recurrent neural network diff --git a/example/gluon/super_resolution/super_resolution.py b/example/gluon/super_resolution/super_resolution.py index 0f2f21f3c0a7..198f6fe0611b 100644 --- a/example/gluon/super_resolution/super_resolution.py +++ b/example/gluon/super_resolution/super_resolution.py @@ -16,19 +16,27 @@ # under the License. from __future__ import print_function -import argparse, tarfile + +import argparse import math import os +import shutil +import sys +import zipfile +from os import path + import numpy as np import mxnet as mx -import mxnet.ndarray as F -from mxnet import gluon +from mxnet import gluon, autograd as ag from mxnet.gluon import nn -from mxnet import autograd as ag -from mxnet.test_utils import download +from mxnet.gluon.contrib import nn as contrib_nn from mxnet.image import CenterCropAug, ResizeAug from mxnet.io import PrefetchingIter +from mxnet.test_utils import download + +this_dir = path.abspath(path.dirname(__file__)) +sys.path.append(path.join(this_dir, path.pardir)) from data import ImagePairIter @@ -51,19 +59,45 @@ batch_size, test_batch_size = opt.batch_size, opt.test_batch_size color_flag = 0 -# get data -dataset_path = "dataset" -dataset_url = "http://www2.eecs.berkeley.edu/Research/Projects/CS/vision/bsds/BSDS300-images.tgz" -def get_dataset(prefetch=False): - image_path = os.path.join(dataset_path, "BSDS300/images") +# Get data +datasets_dir = path.expanduser(path.join("~", ".mxnet", "datasets")) +datasets_tmpdir = path.join(datasets_dir, "tmp") +dataset_url = "https://github.com/BIDS/BSDS500/archive/master.zip" +data_dir = path.expanduser(path.join(datasets_dir, "BSDS500")) +tmp_dir = path.join(data_dir, "tmp") - if not os.path.exists(image_path): - os.makedirs(dataset_path) - file_name = download(dataset_url) - with tarfile.open(file_name) as tar: - for item in tar: - tar.extract(item, dataset_path) - os.remove(file_name) +def get_dataset(prefetch=False): + """Download the BSDS500 dataset and return train and test iters.""" + + if path.exists(data_dir): + print( + "Directory {} already exists, skipping.\n" + "To force download and extraction, delete the directory and re-run." + "".format(data_dir), + file=sys.stderr, + ) + else: + print("Downloading dataset...", file=sys.stderr) + downloaded_file = download(dataset_url, dirname=datasets_tmpdir) + print("done", file=sys.stderr) + + print("Extracting files...", end="", file=sys.stderr) + os.makedirs(data_dir) + os.makedirs(tmp_dir) + with zipfile.ZipFile(downloaded_file) as archive: + archive.extractall(tmp_dir) + shutil.rmtree(datasets_tmpdir) + + shutil.copytree( + path.join(tmp_dir, "BSDS500-master", "BSDS500", "data", "images"), + path.join(data_dir, "images"), + ) + shutil.copytree( + path.join(tmp_dir, "BSDS500-master", "BSDS500", "data", "groundTruth"), + path.join(data_dir, "groundTruth"), + ) + shutil.rmtree(tmp_dir) + print("done", file=sys.stderr) crop_size = 256 crop_size -= crop_size % upscale_factor @@ -72,15 +106,26 @@ def get_dataset(prefetch=False): input_transform = [CenterCropAug((crop_size, crop_size)), ResizeAug(input_crop_size)] target_transform = [CenterCropAug((crop_size, crop_size))] - iters = (ImagePairIter(os.path.join(image_path, "train"), - (input_crop_size, input_crop_size), - (crop_size, crop_size), - batch_size, color_flag, input_transform, target_transform), - ImagePairIter(os.path.join(image_path, "test"), - (input_crop_size, input_crop_size), - (crop_size, crop_size), - test_batch_size, color_flag, - input_transform, target_transform)) + iters = ( + ImagePairIter( + path.join(data_dir, "images", "train"), + (input_crop_size, input_crop_size), + (crop_size, crop_size), + batch_size, + color_flag, + input_transform, + target_transform, + ), + ImagePairIter( + path.join(data_dir, "images", "test"), + (input_crop_size, input_crop_size), + (crop_size, crop_size), + test_batch_size, + color_flag, + input_transform, + target_transform, + ), + ) return [PrefetchingIter(i) for i in iters] if prefetch else iters @@ -90,33 +135,23 @@ def get_dataset(prefetch=False): ctx = [mx.gpu(0)] if opt.use_gpu else [mx.cpu()] -# define model -def _rearrange(raw, F, upscale_factor): - # (N, C * r^2, H, W) -> (N, C, r^2, H, W) - splitted = F.reshape(raw, shape=(0, -4, -1, upscale_factor**2, 0, 0)) - # (N, C, r^2, H, W) -> (N, C, r, r, H, W) - unflatten = F.reshape(splitted, shape=(0, 0, -4, upscale_factor, upscale_factor, 0, 0)) - # (N, C, r, r, H, W) -> (N, C, H, r, W, r) - swapped = F.transpose(unflatten, axes=(0, 1, 4, 2, 5, 3)) - # (N, C, H, r, W, r) -> (N, C, H*r, W*r) - return F.reshape(swapped, shape=(0, 0, -3, -3)) - - -class SuperResolutionNet(gluon.Block): +class SuperResolutionNet(gluon.HybridBlock): def __init__(self, upscale_factor): super(SuperResolutionNet, self).__init__() with self.name_scope(): - self.conv1 = nn.Conv2D(64, (5, 5), strides=(1, 1), padding=(2, 2)) - self.conv2 = nn.Conv2D(64, (3, 3), strides=(1, 1), padding=(1, 1)) - self.conv3 = nn.Conv2D(32, (3, 3), strides=(1, 1), padding=(1, 1)) + self.conv1 = nn.Conv2D(64, (5, 5), strides=(1, 1), padding=(2, 2), activation='relu') + self.conv2 = nn.Conv2D(64, (3, 3), strides=(1, 1), padding=(1, 1), activation='relu') + self.conv3 = nn.Conv2D(32, (3, 3), strides=(1, 1), padding=(1, 1), activation='relu') self.conv4 = nn.Conv2D(upscale_factor ** 2, (3, 3), strides=(1, 1), padding=(1, 1)) - self.upscale_factor = upscale_factor + self.pxshuf = contrib_nn.PixelShuffle2D(upscale_factor) - def forward(self, x): - x = F.Activation(self.conv1(x), act_type='relu') - x = F.Activation(self.conv2(x), act_type='relu') - x = F.Activation(self.conv3(x), act_type='relu') - return _rearrange(self.conv4(x), F, self.upscale_factor) + def hybrid_forward(self, F, x): + x = self.conv1(x) + x = self.conv2(x) + x = self.conv3(x) + x = self.conv4(x) + x = self.pxshuf(x) + return x net = SuperResolutionNet(upscale_factor) metric = mx.metric.MSE() @@ -136,7 +171,7 @@ def test(ctx): avg_psnr += 10 * math.log10(1/metric.get()[1]) metric.reset() avg_psnr /= batches - print('validation avg psnr: %f'%avg_psnr) + print('validation avg psnr: %f' % avg_psnr) def train(epoch, ctx): @@ -168,13 +203,18 @@ def train(epoch, ctx): print('training mse at epoch %d: %s=%f'%(i, name, acc)) test(ctx) - net.save_parameters('superres.params') + net.save_parameters(path.join(this_dir, 'superres.params')) def resolve(ctx): from PIL import Image + if isinstance(ctx, list): ctx = [ctx[0]] - net.load_parameters('superres.params', ctx=ctx) + + img_basename = path.splitext(path.basename(opt.resolve_img))[0] + img_dirname = path.dirname(opt.resolve_img) + + net.load_parameters(path.join(this_dir, 'superres.params'), ctx=ctx) img = Image.open(opt.resolve_img).convert('YCbCr') y, cb, cr = img.split() data = mx.nd.expand_dims(mx.nd.expand_dims(mx.nd.array(y), axis=0), axis=0) @@ -186,7 +226,7 @@ def resolve(ctx): out_img_cr = cr.resize(out_img_y.size, Image.BICUBIC) out_img = Image.merge('YCbCr', [out_img_y, out_img_cb, out_img_cr]).convert('RGB') - out_img.save('resolved.png') + out_img.save(path.join(img_dirname, '{}-resolved.png'.format(img_basename))) if opt.resolve_img: resolve(ctx) diff --git a/python/mxnet/gluon/contrib/nn/__init__.py b/python/mxnet/gluon/contrib/nn/__init__.py index 62440cda27e2..5eb46f6c08ec 100644 --- a/python/mxnet/gluon/contrib/nn/__init__.py +++ b/python/mxnet/gluon/contrib/nn/__init__.py @@ -17,7 +17,7 @@ # coding: utf-8 # pylint: disable=wildcard-import -"""Contrib recurrent neural network module.""" +"""Contributed neural network modules.""" from . import basic_layers diff --git a/python/mxnet/gluon/contrib/nn/basic_layers.py b/python/mxnet/gluon/contrib/nn/basic_layers.py index 56f0809b345f..ebe136e30208 100644 --- a/python/mxnet/gluon/contrib/nn/basic_layers.py +++ b/python/mxnet/gluon/contrib/nn/basic_layers.py @@ -18,8 +18,10 @@ # coding: utf-8 # pylint: disable= arguments-differ """Custom neural network layers in model_zoo.""" + __all__ = ['Concurrent', 'HybridConcurrent', 'Identity', 'SparseEmbedding', - 'SyncBatchNorm'] + 'SyncBatchNorm', 'PixelShuffle1D', 'PixelShuffle2D', + 'PixelShuffle3D'] import warnings from .... import nd, test_utils @@ -238,3 +240,180 @@ def _get_num_devices(self): def hybrid_forward(self, F, x, gamma, beta, running_mean, running_var): return F.contrib.SyncBatchNorm(x, gamma, beta, running_mean, running_var, name='fwd', **self._kwargs) + +class PixelShuffle1D(HybridBlock): + + r"""Pixel-shuffle layer for upsampling in 1 dimension. + + Pixel-shuffling is the operation of taking groups of values along + the *channel* dimension and regrouping them into blocks of pixels + along the ``W`` dimension, thereby effectively multiplying that dimension + by a constant factor in size. + + For example, a feature map of shape :math:`(fC, W)` is reshaped + into :math:`(C, fW)` by forming little value groups of size :math:`f` + and arranging them in a grid of size :math:`W`. + + Parameters + ---------- + factor : int or 1-tuple of int + Upsampling factor, applied to the ``W`` dimension. + + Inputs: + - **data**: Tensor of shape ``(N, f*C, W)``. + Outputs: + - **out**: Tensor of shape ``(N, C, W*f)``. + + Examples + -------- + >>> pxshuf = PixelShuffle1D(2) + >>> x = mx.nd.zeros((1, 8, 3)) + >>> pxshuf(x).shape + (1, 4, 6) + """ + + def __init__(self, factor): + super(PixelShuffle1D, self).__init__() + self._factor = int(factor) + + def hybrid_forward(self, F, x): + """Perform pixel-shuffling on the input.""" + f = self._factor + # (N, C*f, W) + x = F.reshape(x, (0, -4, -1, f, 0)) # (N, C, f, W) + x = F.transpose(x, (0, 1, 3, 2)) # (N, C, W, f) + x = F.reshape(x, (0, 0, -3)) # (N, C, W*f) + return x + + def __repr__(self): + return "{}({})".format(self.__class__.__name__, self._factor) + + +class PixelShuffle2D(HybridBlock): + + r"""Pixel-shuffle layer for upsampling in 2 dimensions. + + Pixel-shuffling is the operation of taking groups of values along + the *channel* dimension and regrouping them into blocks of pixels + along the ``H`` and ``W`` dimensions, thereby effectively multiplying + those dimensions by a constant factor in size. + + For example, a feature map of shape :math:`(f^2 C, H, W)` is reshaped + into :math:`(C, fH, fW)` by forming little :math:`f \times f` blocks + of pixels and arranging them in an :math:`H \times W` grid. + + Pixel-shuffling together with regular convolution is an alternative, + learnable way of upsampling an image by arbitrary factors. It is reported + to help overcome checkerboard artifacts that are common in upsampling with + transposed convolutions (also called deconvolutions). See the paper + `Real-Time Single Image and Video Super-Resolution Using an Efficient + Sub-Pixel Convolutional Neural Network `_ + for further details. + + Parameters + ---------- + factor : int or 2-tuple of int + Upsampling factors, applied to the ``H`` and ``W`` dimensions, + in that order. + + Inputs: + - **data**: Tensor of shape ``(N, f1*f2*C, H, W)``. + Outputs: + - **out**: Tensor of shape ``(N, C, H*f1, W*f2)``. + + Examples + -------- + >>> pxshuf = PixelShuffle2D((2, 3)) + >>> x = mx.nd.zeros((1, 12, 3, 5)) + >>> pxshuf(x).shape + (1, 2, 6, 15) + """ + + def __init__(self, factor): + super(PixelShuffle2D, self).__init__() + try: + self._factors = (int(factor),) * 2 + except TypeError: + self._factors = tuple(int(fac) for fac in factor) + assert len(self._factors) == 2, "wrong length {}".format(len(self._factors)) + + def hybrid_forward(self, F, x): + """Perform pixel-shuffling on the input.""" + f1, f2 = self._factors + # (N, f1*f2*C, H, W) + x = F.reshape(x, (0, -4, -1, f1 * f2, 0, 0)) # (N, C, f1*f2, H, W) + x = F.reshape(x, (0, 0, -4, f1, f2, 0, 0)) # (N, C, f1, f2, H, W) + x = F.transpose(x, (0, 1, 4, 2, 5, 3)) # (N, C, H, f1, W, f2) + x = F.reshape(x, (0, 0, -3, -3)) # (N, C, H*f1, W*f2) + return x + + def __repr__(self): + return "{}({})".format(self.__class__.__name__, self._factors) + + +class PixelShuffle3D(HybridBlock): + + r"""Pixel-shuffle layer for upsampling in 3 dimensions. + + Pixel-shuffling (or voxel-shuffling in 3D) is the operation of taking + groups of values along the *channel* dimension and regrouping them into + blocks of voxels along the ``D``, ``H`` and ``W`` dimensions, thereby + effectively multiplying those dimensions by a constant factor in size. + + For example, a feature map of shape :math:`(f^3 C, D, H, W)` is reshaped + into :math:`(C, fD, fH, fW)` by forming little :math:`f \times f \times f` + blocks of voxels and arranging them in a :math:`D \times H \times W` grid. + + Pixel-shuffling together with regular convolution is an alternative, + learnable way of upsampling an image by arbitrary factors. It is reported + to help overcome checkerboard artifacts that are common in upsampling with + transposed convolutions (also called deconvolutions). See the paper + `Real-Time Single Image and Video Super-Resolution Using an Efficient + Sub-Pixel Convolutional Neural Network `_ + for further details. + + Parameters + ---------- + factor : int or 3-tuple of int + Upsampling factors, applied to the ``D``, ``H`` and ``W`` + dimensions, in that order. + + Inputs: + - **data**: Tensor of shape ``(N, f1*f2*f3*C, D, H, W)``. + Outputs: + - **out**: Tensor of shape ``(N, C, D*f1, H*f2, W*f3)``. + + Examples + -------- + >>> pxshuf = PixelShuffle3D((2, 3, 4)) + >>> x = mx.nd.zeros((1, 48, 3, 5, 7)) + >>> pxshuf(x).shape + (1, 2, 6, 15, 28) + """ + + def __init__(self, factor): + super(PixelShuffle3D, self).__init__() + try: + self._factors = (int(factor),) * 3 + except TypeError: + self._factors = tuple(int(fac) for fac in factor) + assert len(self._factors) == 3, "wrong length {}".format(len(self._factors)) + + def hybrid_forward(self, F, x): + """Perform pixel-shuffling on the input.""" + # `transpose` doesn't support 8D, need other implementation + f1, f2, f3 = self._factors + # (N, C*f1*f2*f3, D, H, W) + x = F.reshape(x, (0, -4, -1, f1 * f2 * f3, 0, 0, 0)) # (N, C, f1*f2*f3, D, H, W) + x = F.swapaxes(x, 2, 3) # (N, C, D, f1*f2*f3, H, W) + x = F.reshape(x, (0, 0, 0, -4, f1, f2*f3, 0, 0)) # (N, C, D, f1, f2*f3, H, W) + x = F.reshape(x, (0, 0, -3, 0, 0, 0)) # (N, C, D*f1, f2*f3, H, W) + x = F.swapaxes(x, 3, 4) # (N, C, D*f1, H, f2*f3, W) + x = F.reshape(x, (0, 0, 0, 0, -4, f2, f3, 0)) # (N, C, D*f1, H, f2, f3, W) + x = F.reshape(x, (0, 0, 0, -3, 0, 0)) # (N, C, D*f1, H*f2, f3, W) + x = F.swapaxes(x, 4, 5) # (N, C, D*f1, H*f2, W, f3) + x = F.reshape(x, (0, 0, 0, 0, -3)) # (N, C, D*f1, H*f2, W*f3) + return x + + def __repr__(self): + return "{}({})".format(self.__class__.__name__, self._factors) diff --git a/tests/python/unittest/test_gluon_contrib.py b/tests/python/unittest/test_gluon_contrib.py index a1cd8ea537d7..6901e8bd12fe 100644 --- a/tests/python/unittest/test_gluon_contrib.py +++ b/tests/python/unittest/test_gluon_contrib.py @@ -19,7 +19,9 @@ import mxnet as mx from mxnet.gluon import contrib from mxnet.gluon import nn -from mxnet.gluon.contrib.nn import Concurrent, HybridConcurrent, Identity, SparseEmbedding +from mxnet.gluon.contrib.nn import ( + Concurrent, HybridConcurrent, Identity, SparseEmbedding, PixelShuffle1D, + PixelShuffle2D, PixelShuffle3D) from mxnet.test_utils import almost_equal from common import setup_module, with_seed, teardown import numpy as np @@ -204,6 +206,89 @@ def test_sparse_embedding(): assert (layer.weight.grad().asnumpy()[:5] == 1).all() assert (layer.weight.grad().asnumpy()[5:] == 0).all() +def test_pixelshuffle1d(): + nchan = 2 + up_x = 2 + nx = 3 + shape_before = (1, nchan * up_x, nx) + shape_after = (1, nchan, nx * up_x) + layer = PixelShuffle1D(up_x) + x = mx.nd.arange(np.prod(shape_before)).reshape(shape_before) + y = layer(x) + assert y.shape == shape_after + assert_allclose( + y.asnumpy(), + [[[0, 3, 1, 4, 2, 5], + [6, 9, 7, 10, 8, 11]]] + ) + +def test_pixelshuffle2d(): + nchan = 2 + up_x = 2 + up_y = 3 + nx = 2 + ny = 3 + shape_before = (1, nchan * up_x * up_y, nx, ny) + shape_after = (1, nchan, nx * up_x, ny * up_y) + layer = PixelShuffle2D((up_x, up_y)) + x = mx.nd.arange(np.prod(shape_before)).reshape(shape_before) + y = layer(x) + assert y.shape == shape_after + # - Channels are reshaped to form 2x3 blocks + # - Within each block, the increment is `nx * ny` when increasing the column + # index by 1 + # - Increasing the block index adds an offset of 1 + # - Increasing the channel index adds an offset of `nx * up_x * ny * up_y` + assert_allclose( + y.asnumpy(), + [[[[ 0, 6, 12, 1, 7, 13, 2, 8, 14], + [18, 24, 30, 19, 25, 31, 20, 26, 32], + [ 3, 9, 15, 4, 10, 16, 5, 11, 17], + [21, 27, 33, 22, 28, 34, 23, 29, 35]], + + [[36, 42, 48, 37, 43, 49, 38, 44, 50], + [54, 60, 66, 55, 61, 67, 56, 62, 68], + [39, 45, 51, 40, 46, 52, 41, 47, 53], + [57, 63, 69, 58, 64, 70, 59, 65, 71]]]] + ) + +def test_pixelshuffle3d(): + nchan = 1 + up_x = 2 + up_y = 1 + up_z = 2 + nx = 2 + ny = 3 + nz = 4 + shape_before = (1, nchan * up_x * up_y * up_z, nx, ny, nz) + shape_after = (1, nchan, nx * up_x, ny * up_y, nz * up_z) + layer = PixelShuffle3D((up_x, up_y, up_z)) + x = mx.nd.arange(np.prod(shape_before)).reshape(shape_before) + y = layer(x) + assert y.shape == shape_after + # - Channels are reshaped to form 2x1x2 blocks + # - Within each block, the increment is `nx * ny * nz` when increasing the + # column index by 1, e.g. the block [[[ 0, 24]], [[48, 72]]] + # - Increasing the block index adds an offset of 1 + assert_allclose( + y.asnumpy(), + [[[[[ 0, 24, 1, 25, 2, 26, 3, 27], + [ 4, 28, 5, 29, 6, 30, 7, 31], + [ 8, 32, 9, 33, 10, 34, 11, 35]], + + [[48, 72, 49, 73, 50, 74, 51, 75], + [52, 76, 53, 77, 54, 78, 55, 79], + [56, 80, 57, 81, 58, 82, 59, 83]], + + [[12, 36, 13, 37, 14, 38, 15, 39], + [16, 40, 17, 41, 18, 42, 19, 43], + [20, 44, 21, 45, 22, 46, 23, 47]], + + [[60, 84, 61, 85, 62, 86, 63, 87], + [64, 88, 65, 89, 66, 90, 67, 91], + [68, 92, 69, 93, 70, 94, 71, 95]]]]] + ) + def test_datasets(): wikitext2_train = contrib.data.text.WikiText2(root='data/wikitext-2', segment='train') wikitext2_val = contrib.data.text.WikiText2(root='data/wikitext-2', segment='validation', From cf46ca459875bd2a3844fe8ec6bc68b744e7f141 Mon Sep 17 00:00:00 2001 From: perdasilva Date: Fri, 15 Feb 2019 00:58:49 +0100 Subject: [PATCH 24/75] Fixes libjpeg-turbo dependecy under Ubuntu 16.04 (#14127) * Fixes libjepgturbo dependency install on ubuntu * Enables libjpeg turbo on cpu build for testing --- ci/docker/install/ubuntu_core.sh | 6 ++++++ ci/docker/runtime_functions.sh | 1 + 2 files changed, 7 insertions(+) diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index 3f8ffb726208..61a4637830da 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -39,6 +39,7 @@ apt-get install -y \ liblapack-dev \ libopenblas-dev \ libopencv-dev \ + libturbojpeg \ libzmq3-dev \ ninja-build \ software-properties-common \ @@ -46,6 +47,11 @@ apt-get install -y \ unzip \ wget +# Use libturbojpeg package as it is correctly compiled with -fPIC flag +# https://github.com/HaxeFoundation/hashlink/issues/147 +ln -s /usr/lib/x86_64-linux-gnu/libturbojpeg.so.0.1.0 /usr/lib/x86_64-linux-gnu/libturbojpeg.so + + # Note: we specify an exact cmake version to work around a cmake 3.10 CUDA 10 issue. # Reference: https://github.com/clab/dynet/issues/1457 mkdir /opt/cmake && cd /opt/cmake diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 93f2d8cc058c..4d9b256634a6 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -365,6 +365,7 @@ build_ubuntu_cpu_openblas() { USE_BLAS=openblas \ USE_MKLDNN=0 \ USE_DIST_KVSTORE=1 \ + USE_LIBJPEG_TURBO=1 \ -j$(nproc) } From 0e08891f855af524926070a54e362b4cdfced714 Mon Sep 17 00:00:00 2001 From: Thomas Delteil Date: Thu, 14 Feb 2019 16:08:37 -0800 Subject: [PATCH 25/75] Fix quote on LBSGD docs (#13975) * change docs lbsgd * fix more --- python/mxnet/optimizer/optimizer.py | 44 +++++++++++++---------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/python/mxnet/optimizer/optimizer.py b/python/mxnet/optimizer/optimizer.py index cb52ac54fdab..a986f271c4b4 100644 --- a/python/mxnet/optimizer/optimizer.py +++ b/python/mxnet/optimizer/optimizer.py @@ -75,12 +75,11 @@ class Optimizer(object): The initial number of updates. multi_precision : bool, optional, default False - Flag to control the internal precision of the optimizer.:: - - False: results in using the same precision as the weights (default), - True: makes internal 32-bit copy of the weights and applies gradients - in 32-bit precision even if actual weights used in the model have lower precision. - Turning this on can improve convergence and accuracy when training with float16. + Flag to control the internal precision of the optimizer. + False: results in using the same precision as the weights (default), + True: makes internal 32-bit copy of the weights and applies gradients + in 32-bit precision even if actual weights used in the model have lower precision. + Turning this on can improve convergence and accuracy when training with float16. param_dict : dict of int -> gluon.Parameter, default None Dictionary of parameter index to gluon.Parameter, used to lookup parameter attributes @@ -541,12 +540,11 @@ class SGD(Optimizer): Default is True. If True, lazy updates are applied \ if the storage types of weight and grad are both ``row_sparse``. multi_precision: bool, optional - Flag to control the internal precision of the optimizer.:: - - False: results in using the same precision as the weights (default), - True: makes internal 32-bit copy of the weights and applies gradients - in 32-bit precision even if actual weights used in the model have lower precision. - Turning this on can improve convergence and accuracy when training with float16. + Flag to control the internal precision of the optimizer. + False: results in using the same precision as the weights (default), + True: makes internal 32-bit copy of the weights and applies gradients + in 32-bit precision even if actual weights used in the model have lower precision. + Turning this on can improve convergence and accuracy when training with float16. """ def __init__(self, momentum=0.0, lazy_update=True, **kwargs): super(SGD, self).__init__(**kwargs) @@ -790,12 +788,11 @@ class LBSGD(Optimizer): momentum : float, optional The momentum value. multi_precision: bool, optional - Flag to control the internal precision of the optimizer.:: - - False: results in using the same precision as the weights (default), - True: makes internal 32-bit copy of the weights and applies gradients - in 32-bit precision even if actual weights used in the model have lower precision. - Turning this on can improve convergence and accuracy when training with float16. + Flag to control the internal precision of the optimizer. + False: results in using the same precision as the weights (default), + True: makes internal 32-bit copy of the weights and applies gradients + in 32-bit precision even if actual weights used in the model have lower precision. + Turning this on can improve convergence and accuracy when training with float16. warmup_strategy: string ('linear', 'power2', 'sqrt'. , 'lars' default : 'linear') warmup_epochs: unsigned, default: 5 @@ -1031,12 +1028,11 @@ class NAG(Optimizer): momentum : float, optional The momentum value. multi_precision: bool, optional - Flag to control the internal precision of the optimizer.:: - - False: results in using the same precision as the weights (default), - True: makes internal 32-bit copy of the weights and applies gradients - in 32-bit precision even if actual weights used in the model have lower precision. - Turning this on can improve convergence and accuracy when training with float16. + Flag to control the internal precision of the optimizer. + False: results in using the same precision as the weights (default), + True: makes internal 32-bit copy of the weights and applies gradients + in 32-bit precision even if actual weights used in the model have lower precision. + Turning this on can improve convergence and accuracy when training with float16. """ def __init__(self, momentum=0.0, **kwargs): super(NAG, self).__init__(**kwargs) From a4e249bee2db7b931ecdd6fed05a94e742e7c3c5 Mon Sep 17 00:00:00 2001 From: Anirudh Date: Thu, 14 Feb 2019 17:07:43 -0800 Subject: [PATCH 26/75] In-place updates for Nadam, Adadelta, Adamax and SGLD (#13960) --- python/mxnet/optimizer/optimizer.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/python/mxnet/optimizer/optimizer.py b/python/mxnet/optimizer/optimizer.py index a986f271c4b4..def2c958ede4 100644 --- a/python/mxnet/optimizer/optimizer.py +++ b/python/mxnet/optimizer/optimizer.py @@ -1091,8 +1091,9 @@ def update(self, index, weight, grad, state): grad = grad * self.rescale_grad if self.clip_gradient is not None: grad = clip(grad, -self.clip_gradient, self.clip_gradient) - weight[:] += - lr/2 * (grad + wd * weight) + normal(0, math.sqrt(lr), shape=weight.shape, - dtype=weight.dtype, ctx=weight.context) + weight[:] += - lr/2 * (grad + wd * weight) + weight[:] += normal(0, math.sqrt(lr), shape=weight.shape, + dtype=weight.dtype, ctx=weight.context) @@ -1372,9 +1373,11 @@ def update(self, index, weight, grad, state): acc_g, acc_delta = state # update g, delta - acc_g[:] = self.rho * acc_g + (1. - self.rho) * grad * grad + acc_g[:] *= self.rho + acc_g[:] += (1. - self.rho) * grad * grad current_delta = sqrt(acc_delta + self.epsilon) / sqrt(acc_g + self.epsilon) * grad - acc_delta[:] = self.rho * acc_delta + (1. - self.rho) * current_delta * current_delta + acc_delta[:] *= self.rho + acc_delta[:] += (1. - self.rho) * current_delta * current_delta # update weight weight[:] -= current_delta + wd * weight @@ -1507,7 +1510,8 @@ def update(self, index, weight, grad, state): # update m_t and u_t m_t, u_t = state - m_t[:] = self.beta1 * m_t + (1. - self.beta1) * grad + m_t[:] *= self.beta1 + m_t[:] += (1. - self.beta1) * grad u_t[:] = maximum(self.beta2 * u_t, NDabs(grad)) # update weight @@ -1570,8 +1574,10 @@ def update(self, index, weight, grad, state): # update m_t and v_t m_t, v_t = state - m_t[:] = self.beta1 * m_t + (1. - self.beta1) * grad - v_t[:] = self.beta2 * v_t + (1. - self.beta2) * grad * grad + m_t[:] *= self.beta1 + m_t[:] += (1. - self.beta1) * grad + v_t[:] *= self.beta2 + v_t[:] += (1. - self.beta2) * grad * grad grad_prime = grad / (1. - self.m_schedule) m_t_prime = m_t / (1. - m_schedule_next) From 8416c563f2bc673f15bf890ea13be2a32afdf3ca Mon Sep 17 00:00:00 2001 From: Ashutosh Dwivedi Date: Fri, 15 Feb 2019 07:00:31 +0530 Subject: [PATCH 27/75] Fix jar path and add missing ones for spark jobs (#14020) * Fix jar path and add missing ones for spark jobs Fix path of jars / add missing jars in spark job remove print, reduce clutter * fixes scalastyle violations * exclude all of javadoc, sources, bundle, and src while searching for jars * simplfied the exclude experession --- .../mxnet/spark/SharedSparkContext.scala | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/scala-package/spark/src/test/scala/org/apache/mxnet/spark/SharedSparkContext.scala b/scala-package/spark/src/test/scala/org/apache/mxnet/spark/SharedSparkContext.scala index 2efd1814bc90..6d36ca51db90 100644 --- a/scala-package/spark/src/test/scala/org/apache/mxnet/spark/SharedSparkContext.scala +++ b/scala-package/spark/src/test/scala/org/apache/mxnet/spark/SharedSparkContext.scala @@ -80,30 +80,27 @@ trait SharedSparkContext extends FunSuite with BeforeAndAfterEach with BeforeAnd System.getProperty("user.dir") } - private def getJarFilePath(root: String): String = { - for (platform <- List("linux-x86_64-cpu", "linux-x86_64-gpu", "osx-x86_64-cpu")) { - val jarFiles = new File(s"$root/$platform/target/").listFiles(new FileFilter { - override def accept(pathname: File) = { - pathname.getAbsolutePath.endsWith(".jar") && - !pathname.getAbsolutePath.contains("javadoc") && - !pathname.getAbsolutePath.contains("sources") - } - }) - if (jarFiles != null && jarFiles.nonEmpty) { - return jarFiles.head.getAbsolutePath + private def findJars(root: String): Array[File] = { + val excludedSuffixes = List("bundle", "src", "javadoc", "sources") + new File(root).listFiles(new FileFilter { + override def accept(pathname: File) = { + pathname.getAbsolutePath.endsWith(".jar") && + excludedSuffixes.forall(!pathname.getAbsolutePath.contains(_)) } + }) + } + + private def getJarFilePath(root: String): String = { + val jarFiles = findJars(s"$root/target/") + if (jarFiles != null && jarFiles.nonEmpty) { + jarFiles.head.getAbsolutePath + } else { + null } - null } private def getSparkJar: String = { - val jarFiles = new File(s"$composeWorkingDirPath/target/").listFiles(new FileFilter { - override def accept(pathname: File) = { - pathname.getAbsolutePath.endsWith(".jar") && - !pathname.getAbsolutePath.contains("javadoc") && - !pathname.getAbsolutePath.contains("sources") - } - }) + val jarFiles = findJars(s"$composeWorkingDirPath/target/") if (jarFiles != null && jarFiles.nonEmpty) { jarFiles.head.getAbsolutePath } else { @@ -111,6 +108,9 @@ trait SharedSparkContext extends FunSuite with BeforeAndAfterEach with BeforeAnd } } + private def getNativeJars(root: String): String = + new File(root).listFiles().map(_.toPath).mkString(",") + protected def buildLeNet(): MXNet = { val workingDir = composeWorkingDirPath val assemblyRoot = s"$workingDir/../assembly" @@ -130,6 +130,8 @@ trait SharedSparkContext extends FunSuite with BeforeAndAfterEach with BeforeAnd protected def buildMlp(): MXNet = { val workingDir = composeWorkingDirPath val assemblyRoot = s"$workingDir/../assembly" + val nativeRoot = s"$workingDir/../native/target/lib" + new MXNet() .setBatchSize(128) .setLabelName("softmax_label") @@ -139,7 +141,7 @@ trait SharedSparkContext extends FunSuite with BeforeAndAfterEach with BeforeAnd .setNumEpoch(10) .setNumServer(1) .setNumWorker(numWorkers) - .setExecutorJars(s"${getJarFilePath(assemblyRoot)},$getSparkJar") + .setExecutorJars(s"${getJarFilePath(assemblyRoot)},$getSparkJar,${getNativeJars(nativeRoot)}") .setJava("java") .setTimeout(0) } From e2c69f30ab1538b2cce746849b00e17fa7e7b573 Mon Sep 17 00:00:00 2001 From: Aaron Markham Date: Fri, 15 Feb 2019 11:43:58 -0800 Subject: [PATCH 28/75] add preprocessed data and pretrained model info; minor format/spelling fixes (#14170) --- example/gluon/lipnet/README.md | 93 ++++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 26 deletions(-) diff --git a/example/gluon/lipnet/README.md b/example/gluon/lipnet/README.md index 894384bc152d..89c27a11330f 100644 --- a/example/gluon/lipnet/README.md +++ b/example/gluon/lipnet/README.md @@ -21,14 +21,16 @@ --- -Gluon implementation of [LipNet: End-to-End Sentence-level Lipreading](https://arxiv.org/abs/1611.01599) +This is a Gluon implementation of [LipNet: End-to-End Sentence-level Lipreading](https://arxiv.org/abs/1611.01599) ![net_structure](asset/network_structure.png) +![sample output](https://user-images.githubusercontent.com/11376047/52533982-d7227680-2d7e-11e9-9f18-c15b952faf0e.png) + ## Requirements - Python 3.6.4 -- MXnet 1.3.0 -- The Required Disk Space: 35Gb +- MXNet 1.3.0 +- Required disk space: 35 GB ``` pip install -r requirements.txt ``` @@ -37,18 +39,58 @@ pip install -r requirements.txt ## The Data - The GRID audiovisual sentence corpus (http://spandh.dcs.shef.ac.uk/gridcorpus/) - - GRID is a large multitalker audiovisual sentence corpus to support joint computational-behavioral studies in speech perception. In brief, the corpus consists of high-quality audio and video (facial) recordings of 1000 sentences spoken by each of 34 talkers (18 male, 16 female). Sentences are of the form "put red at G9 now". The corpus, together with transcriptions, is freely available for research use. + - GRID is a large multi-talker audiovisual sentence corpus to support joint computational-behavioral studies in speech perception. In brief, the corpus consists of high-quality audio and video (facial) recordings of 1000 sentences spoken by each of 34 talkers (18 male, 16 female). Sentences are of the form "put red at G9 now". The corpus, together with transcriptions, is freely available for research use. - Video: (normal)(480 M each) - Each movie has one sentence consist of 6 words. -- Align: word alignments(190 K each) +- Align: word alignments (190 K each) - One align has 6 words. Each word has start time and end time. But this tutorial needs just sentence because of using ctc-loss. - + --- +## Pretrained model +You can train the model yourself in the following sections, you can test a pretrained model's inference, or resume training from the model checkpoint. To work with the provided pretrained model, first download it, then run one of the provided Python scripts for inference (infer.py) or training (main.py). + +* Download the [pretrained model](https://github.com/soeque1/temp_files/files/2848870/epoches_81_loss_15.7157.zip) +* Try inference with the following: + +``` +python infer.py model_path='checkpoint/epoches_81_loss_15.7157' +``` + +* Resume training with the following: + +``` +python main.py model_path='checkpoint/epoches_81_loss_15.7157' +``` + ## Prepare the Data -### (1) Download the data + +You can prepare the data yourself, or you can download preprocessed data. + +### Option 1 - Download the preprocessed data + +There are two download routes provided for the preprocessed data. + +#### Download and untar the data +To download tar zipped files by link, download the following files and extract in a folder called `data` in the root of this example folder. You should have the following structure: +``` +/lipnet/data/align +/lipnet/data/datasets +``` + +* [align files](https://mxnet-public.s3.amazonaws.com/lipnet/data-archives/align.tgz) +* [datasets files](https://mxnet-public.s3.amazonaws.com/lipnet/data-archives/datasets.tgz) + +#### Use AWS CLI to sync the data +To get the folders and files all unzipped with AWS CLI, can use the following command. This will provide the folder structure for you. Run this command from `/lipnet/`: + +``` + aws s3 sync s3://mxnet-public/lipnet/data . +``` + +### Option 2 (part 1)- Download the raw dataset - Outputs - - The Total Moives(mp4): 16GB + - The Total Movies(mp4): 16GB - The Total Aligns(text): 134MB - Arguments - src_path : Path for videos (default='./data/mp4s/') @@ -59,17 +101,17 @@ pip install -r requirements.txt cd ./utils && python download_data.py --n_process=$(nproc) ``` -### (2) Preprocess the Data: Extracting the mouth images from a video and save it. +### Option 2 (part 2) Preprocess the raw dataset: Extracting the mouth images from a video and save it * Using Face Landmark Detection(http://dlib.net/) #### Preprocess (preprocess_data.py) -* If there is no landmark, it download automatically. -* Using Face Landmark Detection, It extract the mouth from a video. +* If there is no landmark, it download automatically. +* Using Face Landmark Detection, It extract the mouth from a video. -- example: +- example: - video: ./data/mp4s/s2/bbbf7p.mpg - - align(target): ./data/align/s2/bbbf7p.align + - align(target): ./data/align/s2/bbbf7p.align : 'sil bin blue by f seven please sil' @@ -85,11 +127,11 @@ Frame 0 | Frame 1 | ... | Frame 74 | :-------------------------:|:-------------------------:|:-------------------------:|:-------------------------: ![](asset/mouth_000.png) | ![](asset/mouth_001.png) | ... | ![](asset/mouth_074.png) -* Save the result images into tgt_path. +* Save the result images into tgt_path. ---- -### How to run +#### How to run the preprocess script - Arguments - src_path : Path for videos (default='./data/mp4s/') @@ -103,16 +145,16 @@ Frame 0 | Frame 1 | ... | Frame 74 | - If you use the multi-processes, you can finish the number of processes faster. - e.g) 9 hours using 6 processes -You can run the preprocessing with just one processor, but this will take a long time (>48 hours). To use all of the available processors, use the following command: +You can run the preprocessing with just one processor, but this will take a long time (>48 hours). To use all of the available processors, use the following command: ``` cd ./utils && python preprocess_data.py --n_process=$(nproc) ``` -## Output: Data Structure +#### Output: Data structure of the preprocessed data ``` -The training data folder should look like : +The training data folder should look like : |--datasets |--s1 @@ -136,10 +178,11 @@ The training data folder should look like : --- ## Training +After you have acquired the preprocessed data you are ready to train the lipnet model. - According to [LipNet: End-to-End Sentence-level Lipreading](https://arxiv.org/abs/1611.01599), four (S1, S2, S20, S22) of the 34 subjects are used for evaluation. The other subjects are used for training. - + - To use the multi-gpu, it is recommended to make the batch size $(num_gpus) times larger. - e.g) 1-gpu and 128 batch_size > 2-gpus 256 batch_size @@ -153,8 +196,8 @@ The training data folder should look like : - dr_rate : Dropout rate(default=0.5) - num_gpus : Num of gpus (if num_gpus is 0, then use cpu) (default=1) - num_workers : Num of workers when generating data (default=0) - - model_path : Path of pretrained model (defalut=None) - + - model_path : Path of pretrained model (default=None) + ``` python main.py ``` @@ -179,8 +222,8 @@ python main.py - num_gpus : Num of gpus (if num_gpus is 0, then use cpu) (default=1) - num_workers : Num of workers when generating data (default=0) - data_type : 'train' or 'valid' (defalut='valid') - - model_path : Path of pretrained model (defalut=None) - + - model_path : Path of pretrained model (default=None) + ``` python infer.py --model_path=$(model_path) ``` @@ -197,7 +240,7 @@ python infer.py --model_path=$(model_path) 'bin blue with e one now', 'lay red at j nine now'] ``` - + ``` [Pred] ['lay green with s zero again', @@ -209,5 +252,3 @@ python infer.py --model_path=$(model_path) 'bin blue with m one now', 'lay red at j nine now'] ``` - - From 549ecde472f208f1c458bfdf585318340a586639 Mon Sep 17 00:00:00 2001 From: Haibin Lin Date: Fri, 15 Feb 2019 12:19:05 -0800 Subject: [PATCH 29/75] support dot(vector, vector) for fp16 inputs on GPU (#14102) * support dot(vector, vector) for fp16 inputs on GPU * add comments * update comments --- src/operator/tensor/dot-inl.h | 3 +- tests/python/unittest/test_operator.py | 73 ++++++++++++++------------ 2 files changed, 41 insertions(+), 35 deletions(-) diff --git a/src/operator/tensor/dot-inl.h b/src/operator/tensor/dot-inl.h index 69c35f85c648..69f87ae42f97 100644 --- a/src/operator/tensor/dot-inl.h +++ b/src/operator/tensor/dot-inl.h @@ -83,7 +83,8 @@ void DotForward_(const nnvm::NodeAttrs& attrs, (outputs[0].type_flag_ == kFloat16 && ctx.run_ctx.ctx.dev_mask() == mshadow::gpu::kDevMask)) << "dot only supports float32/float64 for CPU, and float16/float32/float64 for GPU"; MSHADOW_REAL_TYPE_SWITCH(outputs[0].type_flag_, DType, { - if (inputs[0].ndim() == 1 && inputs[1].ndim() == 1) { + // VectorDot() with fp16 is not supported in mshadow. Dispatch to dot() instead. + if (inputs[0].ndim() == 1 && inputs[1].ndim() == 1 && inputs[0].type_flag_ != kFloat16) { CHECK_NE(req[0], kAddTo) << "AddTo not yet supported"; Tensor out = outputs[0].get(s); VectorDot(out, diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index fc003b2271ef..b0c640bc6e35 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -2631,47 +2631,52 @@ def test_stn_valid_sampling(): ) + target_shape)) -# @haojin2: Getting rid of fixed seed as flakiness could not be reproduced, -# tracked at https://github.com/apache/incubator-mxnet/issues/11714 @with_seed() def test_dot(): - ctx=default_context() + ctx = default_context() dtypes = ['float32', 'float64'] + ndims = [2] if ctx.device_type == 'gpu': dtypes += ['float16'] + ndims += [1] # Test normal dot. - for data_type in dtypes: - for m in range(1, 5): - for k in range(1, 5): - for n in range(1, 5): - a_npy = np.random.normal(0, 1, (m, k)) - a_npy = a_npy.astype(data_type) - b_npy = np.random.normal(0, 1, (k, n)) - b_npy = b_npy.astype(data_type) - c_npy = np.empty((m, n), dtype=data_type) - ograd_npy = np.random.normal(0, 1, (m, n)) - ograd_npy = ograd_npy.astype(data_type) - agrad_npy = np.empty((m, k), dtype=data_type) - bgrad_npy = np.empty((k, n), dtype=data_type) - c_npy[:, :] = np.dot(a_npy[:, :], b_npy[:, :]) - bgrad_npy[:, :] = np.dot(a_npy[:, :].T, ograd_npy[:, :]) - agrad_npy[:, :] = np.dot(ograd_npy[:, :], b_npy[:, :].T) - a = mx.sym.Variable('a', dtype=data_type) - b = mx.sym.Variable('b', dtype=data_type) - c = mx.sym.dot(a, b) - exe = c.simple_bind(ctx=ctx, a=a_npy.shape, b=b_npy.shape) - outputs = exe.forward(is_train=True, a=a_npy, b=b_npy) - assert_almost_equal(outputs[0].asnumpy(), c_npy, - rtol=1e-2 if data_type == 'float16' else 1e-3, - atol=1e-2 if data_type == 'float16' else 1e-3) - exe.backward(out_grads=[mx.nd.array(ograd_npy, mx.cpu()).astype(data_type)]) - assert_almost_equal(exe.grad_dict['a'].asnumpy(), agrad_npy, - rtol=1e-2 if data_type == 'float16' else 1e-3, - atol=1e-2 if data_type == 'float16' else 1e-3) - assert_almost_equal(exe.grad_dict['b'].asnumpy(), bgrad_npy, - rtol=1e-2 if data_type == 'float16' else 1e-3, - atol=1e-2 if data_type == 'float16' else 1e-3) + for ndim in ndims: + for data_type in dtypes: + for m in range(1, 5): + for k in range(1, 5): + if ndim == 1 and k != 1: + pass + for n in range(1, 5): + a_shape = (m, k) if ndim == 2 else (m,) + b_shape = (k, n) if ndim == 2 else (n,) + a_npy = np.random.normal(0, 1, (m, k)) + a_npy = a_npy.astype(data_type) + b_npy = np.random.normal(0, 1, (k, n)) + b_npy = b_npy.astype(data_type) + c_npy = np.empty((m, n), dtype=data_type) + ograd_npy = np.random.normal(0, 1, (m, n)) + ograd_npy = ograd_npy.astype(data_type) + agrad_npy = np.empty((m, k), dtype=data_type) + bgrad_npy = np.empty((k, n), dtype=data_type) + c_npy[:, :] = np.dot(a_npy[:, :], b_npy[:, :]) + bgrad_npy[:, :] = np.dot(a_npy[:, :].T, ograd_npy[:, :]) + agrad_npy[:, :] = np.dot(ograd_npy[:, :], b_npy[:, :].T) + a = mx.sym.Variable('a', dtype=data_type) + b = mx.sym.Variable('b', dtype=data_type) + c = mx.sym.dot(a, b) + exe = c.simple_bind(ctx=ctx, a=a_npy.shape, b=b_npy.shape) + outputs = exe.forward(is_train=True, a=a_npy, b=b_npy) + assert_almost_equal(outputs[0].asnumpy(), c_npy, + rtol=1e-2 if data_type == 'float16' else 1e-3, + atol=1e-2 if data_type == 'float16' else 1e-3) + exe.backward(out_grads=[mx.nd.array(ograd_npy, mx.cpu()).astype(data_type)]) + assert_almost_equal(exe.grad_dict['a'].asnumpy(), agrad_npy, + rtol=1e-2 if data_type == 'float16' else 1e-3, + atol=1e-2 if data_type == 'float16' else 1e-3) + assert_almost_equal(exe.grad_dict['b'].asnumpy(), bgrad_npy, + rtol=1e-2 if data_type == 'float16' else 1e-3, + atol=1e-2 if data_type == 'float16' else 1e-3) # Test dot with transpose flag using gradient checker. def dot_sym(data_type): From 3d0ffd23382ca85b24357c5c03d169855a502672 Mon Sep 17 00:00:00 2001 From: jeremiedb Date: Fri, 15 Feb 2019 15:23:01 -0500 Subject: [PATCH 30/75] R-Package Makefile (#14068) * rpkg Makefile * rpkg makefile fix --- Makefile | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index cc7aebc9fb89..ed450196875f 100644 --- a/Makefile +++ b/Makefile @@ -607,11 +607,7 @@ rpkg: fi mkdir -p R-package/inst/include - cp -rf include/* R-package/inst/include - rm R-package/inst/include/dmlc - rm R-package/inst/include/nnvm - cp -rf 3rdparty/dmlc-core/include/* R-package/inst/include/ - cp -rf 3rdparty/tvm/nnvm/include/* R-package/inst/include + cp -rl include/* R-package/inst/include Rscript -e "if(!require(devtools)){install.packages('devtools', repo = 'https://cloud.r-project.org/')}" Rscript -e "if(!require(roxygen2)||packageVersion('roxygen2') < '6.1.1'){install.packages('roxygen2', repo = 'https://cloud.r-project.org/')}" Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cloud.r-project.org/')); install_deps(pkg='R-package', dependencies = TRUE)" @@ -620,7 +616,7 @@ rpkg: R CMD INSTALL R-package Rscript -e "require(mxnet); mxnet:::mxnet.export('R-package'); warnings()" rm R-package/NAMESPACE - Rscript -e "devtools::document('R-package'); warnings()" + Rscript -e "devtools::document('R-package');warnings()" R CMD INSTALL R-package rpkgtest: From ba97fb6663bf1116c8db492668fbc984e3541612 Mon Sep 17 00:00:00 2001 From: HyperZealot <40650949+HyperZealot@users.noreply.github.com> Date: Fri, 15 Feb 2019 13:28:24 -0800 Subject: [PATCH 31/75] Parallelize CPU version and add GPU version of boolean_mask op (#14090) --- src/operator/contrib/boolean_mask-inl.h | 102 ++++++--------- src/operator/contrib/boolean_mask.cc | 116 ++++++++++++++++- src/operator/contrib/boolean_mask.cu | 165 ++++++++++++++++++++++++ tests/python/unittest/test_operator.py | 2 - 4 files changed, 313 insertions(+), 72 deletions(-) create mode 100644 src/operator/contrib/boolean_mask.cu diff --git a/src/operator/contrib/boolean_mask-inl.h b/src/operator/contrib/boolean_mask-inl.h index ac0681ba927b..775981f76aa8 100644 --- a/src/operator/contrib/boolean_mask-inl.h +++ b/src/operator/contrib/boolean_mask-inl.h @@ -50,83 +50,53 @@ struct BooleanMaskParam : public dmlc::Parameter { } }; +struct BooleanMaskForwardKernel { + template + static void MSHADOW_XINLINE Map(int i, + DType* out, + const DType* data, + const int32_t* idx, + const size_t col_size) { + int row_id = i / col_size; + int col_id = i % col_size; + int32_t prev = (row_id == 0) ? 0 : idx[row_id - 1]; + int32_t curr = idx[row_id]; + if (prev != curr) { + out[prev * col_size + col_id] = data[i]; + } + } +}; + +struct BooleanMaskBackwardKernel { + template + static void MSHADOW_XINLINE Map(int i, + DType* igrad, + const DType* ograd, + const int32_t* idx, + const size_t col_size) { + int row_id = i / col_size; + int col_id = i % col_size; + int32_t prev = (row_id == 0) ? 0 : idx[row_id - 1]; + int32_t curr = idx[row_id]; + if (prev != curr) { + igrad[i] = ograd[prev * col_size + col_id]; + } + } +}; + template inline void BooleanMaskForward(const nnvm::NodeAttrs& attrs, const OpContext &ctx, const std::vector &inputs, const std::vector &req, - const std::vector &outputs) { - // TODO(@junrushao1994): This implementation is a proof-of-concept, - // hence very slow actually. Performance should be improved in the future. - CHECK_EQ(inputs.size(), 2U); - CHECK_EQ(outputs.size(), 1U); - const BooleanMaskParam& param = nnvm::get(attrs.parsed); - const int axis = param.axis; - const NDArray &data = inputs[0]; - const NDArray &idx = inputs[1]; - const NDArray &out = outputs[0]; - CHECK_EQ(axis, 0) << "Not supported yet"; - CHECK_EQ(data.shape()[axis], idx.shape()[0]); - CHECK_EQ(idx.shape().ndim(), 1U); - // count the number of 1s in `idx`, so that we could know the output dimension - size_t valid_num = 0; - MSHADOW_TYPE_SWITCH(idx.dtype(), DType, { - DType* idx_dptr = idx.data().dptr(); - int length = idx.shape()[0]; - for (int i = 0; i < length; i++) { - if (idx_dptr[i]) { - ++valid_num; - } - } - }); - // set the output shape forcefully - TShape s = data.shape(); - s[axis] = valid_num; - const_cast(out).Init(s); - // do the copy - MSHADOW_TYPE_SWITCH(idx.dtype(), DType, { - DType* idx_dptr = idx.data().dptr(); - int length = idx.shape()[0]; - mshadow::Stream *stream = ctx.get_stream(); - for (int i = 0, j = 0; i < length; ++i) { - if (idx_dptr[i]) { - NDArray src = data.At(i); - NDArray dst = out.At(j++); - CHECK(src.shape() == dst.shape()); - mxnet_op::copy(stream, dst.data(), src.data()); - } - } - }); -} + const std::vector &outputs); template inline void BooleanMaskBackward(const nnvm::NodeAttrs& attrs, const OpContext &ctx, const std::vector &inputs, const std::vector &req, - const std::vector &outputs) { - CHECK_EQ(inputs.size(), 3U); - CHECK_EQ(outputs.size(), 2U); - // inputs: {ograd, data, idx} - // outputs: {igrad_data, igrad_idx} - const NDArray& ograd = inputs[0]; - const NDArray& idx = inputs[2]; - const NDArray& igrad_data = outputs[0]; - MSHADOW_TYPE_SWITCH(idx.dtype(), DType, { - DType* idx_dptr = idx.data().dptr(); - int length = idx.shape()[0]; - mshadow::Stream *stream = ctx.get_stream(); - Fill(stream, igrad_data.data(), req[0], 0); - for (int i = 0, j = 0; i < length; ++i) { - if (idx_dptr[i]) { - NDArray src = ograd.At(j++); - NDArray dst = igrad_data.At(i); - CHECK(src.shape() == dst.shape()); - mxnet_op::copy(stream, dst.data(), src.data()); - } - } - }); -} + const std::vector &outputs); } // namespace op } // namespace mxnet diff --git a/src/operator/contrib/boolean_mask.cc b/src/operator/contrib/boolean_mask.cc index 7fd66bc321bf..18ba8c3fdcf6 100644 --- a/src/operator/contrib/boolean_mask.cc +++ b/src/operator/contrib/boolean_mask.cc @@ -28,7 +28,6 @@ namespace op { DMLC_REGISTER_PARAMETER(BooleanMaskParam); - bool BooleanMaskType(const nnvm::NodeAttrs& attrs, std::vector *in_attrs, std::vector *out_attrs) { @@ -75,9 +74,116 @@ bool BooleanMaskBackStorageType(const nnvm::NodeAttrs& attrs, return true; } +struct BooleanMaskForwardCPUKernel { + template + static void Map(int i, + DType* out, + const DType* data, + const int32_t* idx, + const size_t col_size) { + // i is row id already + int32_t prev = (i == 0) ? 0 : idx[i - 1]; + int32_t curr = idx[i]; + if (prev != curr) { + std::memcpy(out + prev * col_size, data + i * col_size, col_size * sizeof(DType)); + } + } +}; + +struct BooleanMaskBackwardCPUKernel { + template + static void Map(int i, + DType* igrad, + const DType* ograd, + const int32_t* idx, + const size_t col_size) { + // i is row id already + int32_t prev = (i == 0) ? 0 : idx[i - 1]; + int32_t curr = idx[i]; + if (prev != curr) { + std::memcpy(igrad + i * col_size, ograd + prev * col_size, col_size * sizeof(DType)); + } + } +}; + +template<> +inline void BooleanMaskForward(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs) { + CHECK_EQ(inputs.size(), 2U); + CHECK_EQ(outputs.size(), 1U); + const BooleanMaskParam& param = nnvm::get(attrs.parsed); + const int axis = param.axis; + const NDArray &data = inputs[0]; + const NDArray &idx = inputs[1]; + const NDArray &out = outputs[0]; + CHECK_EQ(axis, 0) << "Not supported yet"; + CHECK_EQ(data.shape()[axis], idx.shape()[0]); + CHECK_EQ(idx.shape().ndim(), 1U); + // count the number of 1s in `idx`, so that we could know the output dimension + size_t idx_size = idx.shape()[0]; + std::vector prefix_sum(idx_size, 0); + size_t valid_num = 0; + // Calculate prefix sum + MSHADOW_TYPE_SWITCH(idx.dtype(), DType, { + DType* idx_dptr = idx.data().dptr(); + for (size_t i = 0; i < idx_size; i++) { + prefix_sum[i] = (i == 0) ? 0 : prefix_sum[i - 1]; + prefix_sum[i] += (idx_dptr[i]) ? 1 : 0; + } + valid_num = prefix_sum[idx_size - 1]; + }); + // set the output shape forcefully + TShape s = data.shape(); + s[axis] = valid_num; + const_cast(out).Init(s); + // do the copy + MSHADOW_TYPE_SWITCH(data.dtype(), DType, { + size_t input_size = data.shape().Size(); + size_t col_size = input_size / idx_size; + mshadow::Stream *stream = ctx.get_stream(); + mxnet_op::Kernel::Launch( + stream, idx_size, out.data().dptr(), data.data().dptr(), + prefix_sum.data(), col_size); + }); +} + +template<> +inline void BooleanMaskBackward(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs) { + CHECK_EQ(inputs.size(), 3U); + CHECK_EQ(outputs.size(), 2U); + // inputs: {ograd, data, idx} + // outputs: {igrad_data, igrad_idx} + const NDArray& ograd = inputs[0]; + const NDArray& idx = inputs[2]; + const NDArray& igrad_data = outputs[0]; + MSHADOW_TYPE_SWITCH(igrad_data.dtype(), DType, { + MSHADOW_TYPE_SWITCH(idx.dtype(), IType, { + size_t input_size = igrad_data.shape().Size(); + size_t idx_size = idx.shape()[0]; + size_t col_size = input_size / idx_size; + std::vector prefix_sum(idx_size, 0); + IType* idx_dptr = idx.data().dptr(); + for (size_t i = 0; i < idx_size; i++) { + prefix_sum[i] = (i == 0) ? 0 : prefix_sum[i - 1]; + prefix_sum[i] += (idx_dptr[i]) ? 1 : 0; + } + mshadow::Stream *stream = ctx.get_stream(); + mxnet_op::Kernel::Launch( + stream, idx_size, igrad_data.data().dptr(), ograd.data().dptr(), + prefix_sum.data(), col_size); + }); + }); +} + NNVM_REGISTER_OP(_contrib_boolean_mask) .describe(R"code( -Experimental CPU-only support for boolean masking. Given an n-d NDArray data, and a 1-d NDArray index, the operator produces an un-predeterminable shaped n-d NDArray out, which stands for the rows in x where the corresonding element in index is non-zero. @@ -94,12 +200,14 @@ which stands for the rows in x where the corresonding element in index is non-ze .set_attr_parser(ParamParser) .set_num_inputs(2) .set_num_outputs(1) +.set_attr("FListInputNames", + [](const NodeAttrs& attrs) { + return std::vector{"data", "index"}; + }) .set_attr("FInferType", BooleanMaskType) .set_attr("FComputeEx", BooleanMaskForward) .set_attr("FInferStorageType", BooleanMaskStorageType) .set_attr("FGradient", ElemwiseGradUseIn{"_backward_contrib_boolean_mask"}) -.set_attr("FListInputNames", [](const NodeAttrs& attrs) { - return std::vector{"data", "index"};}) .add_argument("data", "NDArray-or-Symbol", "Data") .add_argument("index", "NDArray-or-Symbol", "Mask") .add_arguments(BooleanMaskParam::__FIELDS__()); diff --git a/src/operator/contrib/boolean_mask.cu b/src/operator/contrib/boolean_mask.cu new file mode 100644 index 000000000000..25a781ceec4b --- /dev/null +++ b/src/operator/contrib/boolean_mask.cu @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/*! + * Copyright (c) 2018 by Contributors + * \file boolean_mask.cu +*/ + +#include "./boolean_mask-inl.h" +#include + +namespace mxnet { +namespace op { + +template<> +inline void BooleanMaskForward(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs) { + using namespace mshadow; + CHECK_EQ(inputs.size(), 2U); + CHECK_EQ(outputs.size(), 1U); + const BooleanMaskParam& param = nnvm::get(attrs.parsed); + const int axis = param.axis; + const NDArray &data = inputs[0]; + const NDArray &idx = inputs[1]; + const NDArray &out = outputs[0]; + CHECK_EQ(axis, 0) << "Not supported yet"; + CHECK_EQ(data.shape()[axis], idx.shape()[0]); + CHECK_EQ(idx.shape().ndim(), 1U); + Stream* s = ctx.get_stream(); + // count the number of 1s in `idx`, so that we could know the output dimension + size_t idx_size = idx.shape()[0]; + int32_t valid_num = 0; + int32_t* prefix_sum = nullptr; + void* d_temp_storage = nullptr; + size_t temp_storage_bytes = 0; + // Calculate total temporary memory size + cub::DeviceScan::InclusiveSum(d_temp_storage, + temp_storage_bytes, + prefix_sum, + prefix_sum, + idx_size, + Stream::GetStream(s)); + size_t buffer_size = idx_size * sizeof(int32_t); + temp_storage_bytes += buffer_size; + // Allocate memory on GPU and allocate pointer + Tensor workspace = + ctx.requested[0].get_space_typed(Shape1(temp_storage_bytes), s); + prefix_sum = reinterpret_cast(workspace.dptr_); + d_temp_storage = workspace.dptr_ + buffer_size; + MSHADOW_TYPE_SWITCH(idx.dtype(), IType, { + mxnet_op::Kernel::Launch( + s, idx.shape()[0], prefix_sum, idx.data().dptr()); + }); + // Calculate prefix sum + cub::DeviceScan::InclusiveSum(d_temp_storage, + temp_storage_bytes, + prefix_sum, + prefix_sum, + idx_size, + Stream::GetStream(s)); + CUDA_CALL(cudaMemcpy(&valid_num, &prefix_sum[idx_size - 1], sizeof(int32_t), + cudaMemcpyDeviceToHost)); + CHECK(valid_num > 0) << "boolean_mask behavior not defined when all masks are 0"; + // Set the output shape forcefully + TShape data_shape = data.shape(); + data_shape[axis] = valid_num; + const_cast(out).Init(data_shape); + size_t input_size = data.shape().Size(); + size_t col_size = input_size / idx.shape()[0]; + // Do the copy + MSHADOW_TYPE_SWITCH(out.dtype(), DType, { + mxnet_op::Kernel::Launch( + s, input_size, out.data().dptr(), data.data().dptr(), prefix_sum, col_size); + }); +} + +template<> +inline void BooleanMaskBackward(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs) { + using namespace mshadow; + CHECK_EQ(inputs.size(), 3U); + CHECK_EQ(outputs.size(), 2U); + // inputs: {ograd, data, idx} + // outputs: {igrad_data, igrad_idx} + const NDArray& ograd = inputs[0]; + const NDArray& idx = inputs[2]; + const NDArray& igrad_data = outputs[0]; + Stream* s = ctx.get_stream(); + // Count the number of 1s in `idx`, so that we could know the output dimension + size_t idx_size = idx.shape()[0]; + int32_t* prefix_sum = nullptr; + void* d_temp_storage = nullptr; + size_t temp_storage_bytes = 0; + // Calculate total temporary memory size + cub::DeviceScan::InclusiveSum(d_temp_storage, + temp_storage_bytes, + prefix_sum, + prefix_sum, + idx_size, + Stream::GetStream(s)); + size_t buffer_size = idx_size * sizeof(int32_t); + temp_storage_bytes += buffer_size; + // Allocate memory on GPU and allocate pointer + Tensor workspace = + ctx.requested[0].get_space_typed(Shape1(temp_storage_bytes), s); + prefix_sum = reinterpret_cast(workspace.dptr_); + d_temp_storage = workspace.dptr_ + buffer_size; + MSHADOW_TYPE_SWITCH(idx.dtype(), IType, { + mxnet_op::Kernel::Launch( + s, idx.shape()[0], prefix_sum, idx.data().dptr()); + }); + // Calculate prefix sum + cub::DeviceScan::InclusiveSum(d_temp_storage, + temp_storage_bytes, + prefix_sum, + prefix_sum, + idx_size, + Stream::GetStream(s)); + size_t input_size = igrad_data.shape().Size(); + size_t col_size = input_size / idx_size; + // Backward pass + MSHADOW_TYPE_SWITCH(igrad_data.dtype(), DType, { + mxnet_op::Kernel::Launch( + s, input_size, igrad_data.data().dptr(), ograd.data().dptr(), + prefix_sum, col_size); + }); +} + +NNVM_REGISTER_OP(_contrib_boolean_mask) +.set_attr("FResourceRequest", + [](const NodeAttrs& attrs) { + return std::vector{ResourceRequest::kTempSpace}; + }) +.set_attr("FComputeEx", BooleanMaskForward); + +NNVM_REGISTER_OP(_backward_contrib_boolean_mask) +.set_attr("FResourceRequest", + [](const NodeAttrs& attrs) { + return std::vector{ResourceRequest::kTempSpace}; + }) +.set_attr("FComputeEx", BooleanMaskBackward); + +} // namespace op +} // namespace mxnet diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index b0c640bc6e35..a9b9cc8cf704 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -4880,8 +4880,6 @@ def test_index_copy(): @with_seed() def test_boolean_mask(): - if default_context().device_type != 'cpu': - return data = mx.nd.array([[1, 2, 3],[4, 5, 6],[7, 8, 9]]) index = mx.nd.array([0, 1, 0]) data.attach_grad() From da9afe1e69b10af983fd2a700f9dc7e3cf76fc89 Mon Sep 17 00:00:00 2001 From: Lanking Date: Fri, 15 Feb 2019 14:00:37 -0800 Subject: [PATCH 32/75] add Qing's Key to master (#14180) --- KEYS | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/KEYS b/KEYS index 03b3ed40ccf6..7b78ea97e3ea 100644 --- a/KEYS +++ b/KEYS @@ -599,3 +599,93 @@ a4LYL628Ksuv1Yxn/Uhb5nDPxU5RKRDeogn07wtaqSG12T9pcIwmExW1RN5kvXc2 3Os= =XL0V -----END PGP PUBLIC KEY BLOCK----- + +pub rsa2048 2019-02-15 [SC] [expires: 2021-02-14] + 0812952358B12DC30536E7E0C06916C3AB88ABFE +uid [ultimate] Qing Lan +sig 3 C06916C3AB88ABFE 2019-02-15 Qing Lan +sub rsa2048 2019-02-15 [E] [expires: 2021-02-14] +sig C06916C3AB88ABFE 2019-02-15 Qing Lan + +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBFwb8GEBCADjmi5ZXihPfPbLxhNpbD4HVdT7xGEQgfgkeTA4TdFyBP81zM0F +dTHPQOhHzPGkHrVwUPt3ir2hS46q3L8wni3VRkUU8KPbqbS/a7Wl7LHFFS0lU36J +3uQLElZOFITlaL1dl7cIv+c8xCfmOPlmEtNAPIB26sIM5qzc5l4xvNf1H0Oq0wo6 +VKCsYb4el4nys2U3UBYVQjGyBEwwemHQmFPKg6a2bc/2UhWn4Z+//g0hzIpYtT/S +jua16r5SHy6BUtFGWfU6LQIwmxqc0TNqRdkDU0QY0A+nT6cgx6ghp/qxoLOhgken +Uw8rutC/oFg2VzS9yVsJNrkbQq5Fl/Mz/wqNABEBAAG0HWxhbmtpbmcgPGxhbmtp +bmdAbGFua2luZy5uZXQ+iQFUBBMBCAA+FiEECukpOOn0VXTzKWFpZ3gsZdQlyvsF +Alwb8GECGwMFCQPCZwAFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQZ3gsZdQl +yvtmiAgAnFxzWwPSoHZW9U8ejNtLXxqKmfrUVHDJKVVE0zZPEs9kEk95YZ99jqUS +LagTZwdQOGCJe6Fc9XESW5KdzvnwQyJKwGPCIYDVA/CRuGDbs+tY6HCZ7V1DMGYm +EJsQEzdqWKaNLEVNk1LB3uizEjTS3BIVaVZ0+uovv2joM1mgHX9obGZAZRpBm7zG +TsDwQMuAaWpgnh+BQ9b8O0m6Mdlqh4lAHJmqLdwIz3rD1SLKst0WenjVtyBE1k5t +gBSVfQeMThhcbPITUTLdhAmc27QrsJdDdsH42CVhIJMW6aQnBnEKE3QB+C6DzpTi +UIKqPg99PXmBc9BvlugrZRM4KYiqOLkBDQRcG/BhAQgAzS6PeCCIwImwqSRRnwta +R/7ZbjhMDbnbiHhBKjy08vt2tR8cRK8X7utVwd96u0b3a6J4k0VzMcDP7wBYiY0x +GDbEVsmeXxiqh3+Dv2R+/CBebbswlMJqG+imEaGAAA1bk8U8RrjxuKAUOlyVjEaG +m9flPKOo33SK7UTujfnpSZhA6s0wzRSMn3/26c6+/9vpo8pnqooiD+K7eMhIUfjB ++hFRDJxrEPg9pj3hCJhXA1aCIMrq8jjkm+8PyDQWdt9TdnHLD+b5X+DKNNBpZsIk +IxNTqxNNLan4JZ+70z5m68Lqy4EEHFoHwhn8IFjxOMiJ+6MuP32UdYmvUD3owc8y +nwARAQABiQE8BBgBCAAmFiEECukpOOn0VXTzKWFpZ3gsZdQlyvsFAlwb8GECGwwF +CQPCZwAACgkQZ3gsZdQlyvvYwQf/ctlGApkvSxmhKgCTPBCrudGDpO3QsEF+bR6y +w9WvyvniCt6t5M2A+QPOCqdBvNaP/4wP1H2XghKr9XZcTBlxAJc6pdwxQ6IH6SD8 +syEDVTp0oiQytuyWeNoTM9bOFsliCkjiAVphTDTuVrNEgqdnTMo2cDZgry6gwD0Q +1ZOejcb7kIjiThzaKmgpPnnDNkiK2j25fiNhpiIzEVryLKoXvyoj//C/p8lm9KRk +/2bzRm84uoRsUlrr6I5qizakoODkh5+DlzAaes1fv8ED11zzrlIZjh1vBBgDd3QT +jurgAMn1+WYXhnTVUw8+JjWbmZX4NqJJGZyozkF7rgzI3A9CPJkBDQRcLmPJAQgA +6P5AdfBp4QJKfJC96BjDIrXAFXTytD5wxBsyl1WUcq9vnkX6NZGEtTgsuR/F+FBt +Za7fuDRf+ZT5J7sFQbKFJKRvR8shg1dqnIsoztBD0ESC46trD/YZrisDb09JUXN+ +zvha7jUisA7/pVwfenQM8CpnZJiuKIDU5qCTUr7qOjds6g9aCtUSOK41xLAn1y9b +hKJko4xtjUQu6hXjRXXKltl1N0X5TQCw3sR105xTTWx9nAgFu7cc1uGGuOPCpSms +FPkXFMz+75ff9eXtnmpM+9ZxE8c0jTN7p3U16nd2spsQOqQd83nKb3AbfmWrUYC6 +lmiXGlyOqf8fXSSj/d1ZWwARAQABtB5RaW5nIExhbiA8bGFua2luZzUyMEBsaXZl +LmNvbT6JAVQEEwEIAD4WIQSEDPQiKrv4QoSvRxUvo3RD/I+Y+gUCXC5jyQIbAwUJ +A8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRAvo3RD/I+Y+vjlB/4h3trK +2wGFn396m/jqLqBGObCrOTQYJVycfKdndTEvzIm5/U1gU0Q2BzP25k3gYQjjYKDB +ZN5ETrvRa+NUHzK9p4L4mEQpyWI1WR7jJ19Qg3wfjXW2lOJ8hVvGDILKm7JHEOFb +xf8bUsIftAJ+eDs0dFkDEZT+VLGIXWghd236oUy/uYKL0nLofyFZL3IHiBRZkZQf +6nW+wuYlNXJ+Gy8ATUOKmqbhKScOYLT88nvd6XK8ht3agvATTiEZNa7uWeSOZqp/ +Sx10jUKvKif1L3KiRADb461zQdijx1VOpj3OZPKwCViag14TGXot/YsjAHLCROPg +RH7Ey+WHqMzrkI/EuQENBFwuY8kBCADG74gBceYTyUZbwP6y1xKElXacZ4r2aovT +xoyEXGGovkVjR30GT4XWpKBbqFk5S+QtRnJJGCMeVeKbruZc2Jv5cNe5WL8t20SU +shkBi89h2ZWOgw5c8h+5axjyJfPzWuq0PoXZZ0poFVuSojEFuQKotBWqalri3SKu +E7dUlPQ1JzCj60dX03Lqb2fZZEuwNTf3VwXZZHCgRKw0jhvlr0yfZVcIx4C1Vizo +rkmgZoKbTTkcjBwG4xU+QAwtZqtqGUcZsGwGRD/VPYoDwVknTWJN9JwGasM2gdtG +y0IvhjHfhahZXPyiDsUpkvHyrjaKUwF9qkNLlnmV0LxRJYfNYkk1ABEBAAGJATwE +GAEIACYWIQSEDPQiKrv4QoSvRxUvo3RD/I+Y+gUCXC5jyQIbDAUJA8JnAAAKCRAv +o3RD/I+Y+ql8CADZJxKXAEhBN6ao0+OE2RM/+IPUgIuW/B33KxYN/SewVaUeZvIe +Nu3TWLiFa4eTOjc2hNZxRmQRtpmX7faJaynuz/bsfdtdsscuWxNUUiF3Tdd9/Eon +eFaywH8as9uFwYLRmfdb5cvst97XofQOJBEfolOgyGOEEuEVxs6bFa43/SSYCliE +MjhxubT8NqQmIsHGQ4nltxCbmVZNKi7eAjms+QkWX5bqWMqXF5hJisagXvO1Jr75 +ncoq3Wp+X1b+9x8KDGCVd1F5+d0OksPrnNIQVOS8TMw6bJzXLw4yQni+1ZtYTw8p +9eG+gakLwBnBvqs+7HmuvPNELZX5SsKfwRYzmQENBFxnEhcBCADFLKeuIwb8XnYd +MsAEbQIyueHsON04/XJVvULVHlbpF7X4XyUU2aHxkydFN5yGtJZjE7/BXrPSeoK8 +0qtTblb7ZugUgnyhXaN97RqdD6Lv6kha6EFSylUG9Wyfi1Onr0zlgqQ5TwbaTbLJ +UuzN5d7UIPNFZBrWgvH6AC982M3kMGBWEPRiJgCs/k601oX84g6HbynuxaySts1W +N8mC1TjzVuTRq6Cbfr1RAkNcj3mz66DpZLnHsBUPuQ5u0gJTKZyEGRxtT02eXpMI +61eP+79hnAwqZds9BXwO+3cCx/x3qJRLNDpR4mPq9QqETkqTIQd6JPjWPRh0OQDK +300hDmizABEBAAG0HVFpbmcgTGFuIDxsYW5raW5nQGFwYWNoZS5vcmc+iQFUBBMB +CAA+FiEECBKVI1ixLcMFNufgwGkWw6uIq/4FAlxnEhcCGwMFCQPCZwAFCwkIBwIG +FQoJCAsCBBYCAwECHgECF4AACgkQwGkWw6uIq/5KDgf/S8XUCzm+aVJUAcx9BuQV +SzvG8C1Bf/XrYXcm3NxEOMejdkt8FAnaPt9dZM3U8CaMu1a90iiuLyvr3aJvb7/K +/WuaK79jFB+zz8bifQANXgwAQPiUjBU/8FNOjiEywaSYwIvy8SMo5lLs2edV4Y8g +J73BU/WID3aH/BK4sclAs3brNY5le3B2GiAXhqCWB567ZTlc6vpf1DBaivVX8J0H +cEBIDRuitSkWrFYGmx4Ddr7JebkgNBctJaZSkquHxxORLWfFnEqzUhSzKwdZUTTO +bfg31btuFHTiEuQFM+jYGzYBkwW5dlJMRKPTFGg712VwvtbeE6ZkF3AQFARaOlPx +MrkBDQRcZxIXAQgAxRPIrPzi2sv7OpOzKyZZItqSPYVncDPIV4FSMqXbcB0MHT+I +g3tQJFmmFo5q53vdyGwzUmU9ft4jxkif+IQS+pvmXn1q3WNm/+t62ylM6S1hV4sW +ZgJAfrmfID19XtdtyNKSdRIWG+h/q/Mi8clT4RO1/jMnimDNw1KHOBeC3TncJjCn +lpkGVHJMw/S7+odOZmKaXIevJiNpkeXqY+A25w92MNEx7mSn2icNIfhODEdk8GJH +P6kbk2c1q+GPeS9wJap5XCASVSfVlsIPFKm/s9O0OUJ+hmQJq5yXDuEjxE5FyFnq +5uWcYlS3OU7JAJMwaPNVShRzy4+3J4e/F02DjwARAQABiQE8BBgBCAAmFiEECBKV +I1ixLcMFNufgwGkWw6uIq/4FAlxnEhcCGwwFCQPCZwAACgkQwGkWw6uIq/5pCAgA +tZYnewxZPDQN/VHHT4yyYXf67SLWoygiQ9epsIpmvQYJFREbvKlVKUxudSx0CZWc +DTaIup6stXPgB33pmWvV3ia1B+2kNcBPuXGoilHBjmG1oMp2/9Y9NjbFwmsZUo4V +TXGEQvhthiZUelBTWiTrCKYYOl7VEthXpB4OoHJB7wZ+PE4aOmZdArK6cT5/fATT +n4aiPNGpG7CDmCNnGMJgNYEEbqe1RQ7B4xwmNmGJvdVJRsTfy5557hZNfIfVkdES +QTXMfTPP627GwzHQXTdAn9CSGW5FkaSHTVTCZhalBHhAFMDg86ZGUxZDYwhf3s6W +44liPzisQFRxRFOwEubvmw== +=dQvb +-----END PGP PUBLIC KEY BLOCK----- From b0ab8d54c1ea32ff7d3808d88fd288ee10824951 Mon Sep 17 00:00:00 2001 From: perdasilva Date: Fri, 15 Feb 2019 23:01:33 +0100 Subject: [PATCH 33/75] Fixes spelling (#14168) --- tests/jenkins/run_test_installation_docs.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/jenkins/run_test_installation_docs.sh b/tests/jenkins/run_test_installation_docs.sh index 4f62865adac0..954b370ae50b 100755 --- a/tests/jenkins/run_test_installation_docs.sh +++ b/tests/jenkins/run_test_installation_docs.sh @@ -255,12 +255,11 @@ function set_instruction_set() { # $ git clone --recursive https://github.com/apache/incubator-mxnet # $ cd incubator-mxnet # if these commands get executed in the jenkins job, we will be testing the build from source instructions -# against the master branch and not against the version of the repository that Jenkins checksout for testing. +# against the master branch and not against the version of the repository that Jenkins checks out for testing. # This presents a particularly big problem for the version branches and their nightly builds. Because, # we would, in effect, be testing the build from source instructions for one version of MXNet against # the master branch. -# in this function we target the commands cited in the example above, but leave it open for expantion -# in the future. +# in this function we target the commands cited in the example above. # See also gh issue: https://github.com/apache/incubator-mxnet/issues/13800 function filter_build_commands() { filtered_build_commands="${1}" From 4b527b681f01bff02a770c1b49a3b5c9278cf4a0 Mon Sep 17 00:00:00 2001 From: Da Zheng Date: Fri, 15 Feb 2019 18:23:01 -0800 Subject: [PATCH 34/75] [MXNET-766] add dynamic_unroll RNN for HybridBlock (#11948) * add contrib unroll. * reenable some tests. * fix a bug. * fix lint. * fix a bug. * support diff layouts. * update doc. * use a diff default layout. * remove _contrib_format_sequence. * fix lint. * rename. --- python/mxnet/gluon/contrib/rnn/rnn_cell.py | 115 ++++++++++++++++++++ tests/python/unittest/test_gluon_contrib.py | 95 +++++++++++++++- 2 files changed, 209 insertions(+), 1 deletion(-) diff --git a/python/mxnet/gluon/contrib/rnn/rnn_cell.py b/python/mxnet/gluon/contrib/rnn/rnn_cell.py index 0cbc9eaac375..3bd8e7810978 100644 --- a/python/mxnet/gluon/contrib/rnn/rnn_cell.py +++ b/python/mxnet/gluon/contrib/rnn/rnn_cell.py @@ -22,6 +22,7 @@ from ...rnn import BidirectionalCell, SequentialRNNCell, ModifierCell, HybridRecurrentCell from ...rnn.rnn_cell import _format_sequence, _get_begin_state, _mask_sequence_variable_length from ... import tensor_types +from ....base import _as_list class VariationalDropoutCell(ModifierCell): """ @@ -320,3 +321,117 @@ def hybrid_forward(self, F, inputs, states, i2h_weight, return next_r, [next_r, next_c] # pylint: enable= arguments-differ + + +def dynamic_unroll(cell, inputs, begin_state, drop_inputs=0, drop_outputs=0, + layout='TNC', valid_length=None): + """Unrolls an RNN cell across time steps. + + Currently, 'TNC' is a preferred layout. unroll on the input of this layout + runs much faster. + + Parameters + ---------- + cell : an object whose base class is RNNCell. + The RNN cell to run on the input sequence. + inputs : Symbol + It should have shape (batch_size, length, ...) if `layout` is 'NTC', + or (length, batch_size, ...) if `layout` is 'TNC'. + begin_state : nested list of Symbol + The initial states of the RNN sequence. + drop_inputs : float, default 0. + The dropout rate for inputs. Won't apply dropout if it equals 0. + drop_outputs : float, default 0. + The dropout rate for outputs. Won't apply dropout if it equals 0. + layout : str, optional + `layout` of input symbol. Only used if inputs + is a single Symbol. + valid_length : Symbol, NDArray or None + `valid_length` specifies the length of the sequences in the batch without padding. + This option is especially useful for building sequence-to-sequence models where + the input and output sequences would potentially be padded. + If `valid_length` is None, all sequences are assumed to have the same length. + If `valid_length` is a Symbol or NDArray, it should have shape (batch_size,). + The ith element will be the length of the ith sequence in the batch. + The last valid state will be return and the padded outputs will be masked with 0. + Note that `valid_length` must be smaller or equal to `length`. + + Returns + ------- + outputs : Symbol + the output of the RNN from this unrolling. + + states : list of Symbol + The new state of this RNN after this unrolling. + The type of this symbol is same as the output of `begin_state`. + + Examples + -------- + >>> seq_len = 3 + >>> batch_size = 2 + >>> input_size = 5 + >>> cell = mx.gluon.rnn.LSTMCell(input_size, prefix='rnn_') + >>> cell.initialize(ctx=mx.cpu()) + >>> rnn_data = mx.nd.normal(loc=0, scale=1, shape=(seq_len, batch_size, input_size)) + >>> state_shape = (batch_size, input_size) + >>> states = [mx.nd.normal(loc=0, scale=1, shape=state_shape) for i in range(2)] + >>> valid_length = mx.nd.array([2, 3]) + >>> output, states = mx.gluon.contrib.rnn.rnn_cell.dynamic_unroll(cell, rnn_data, states, + valid_length=valid_length, + layout='TNC') + >>> print(output) + [[[ 0.00767238 0.00023103 0.03973929 -0.00925503 -0.05660512] + [ 0.00881535 0.05428379 -0.02493718 -0.01834097 0.02189514]] + [[-0.00676967 0.01447039 0.01287002 -0.00574152 -0.05734247] + [ 0.01568508 0.02650866 -0.04270559 -0.04328435 0.00904011]] + [[ 0. 0. 0. 0. 0. ] + [ 0.01055336 0.02734251 -0.03153727 -0.03742751 -0.01378113]]] + + """ + + # Merge is always True, so we don't need length. + inputs, axis, F, _ = _format_sequence(0, inputs, layout, True) + if axis != 0: + axes = list(range(len(layout))) + tmp = axes[0] + axes[0] = axes[axis] + axes[axis] = tmp + inputs = F.transpose(inputs, axes=axes) + states = begin_state + + if drop_inputs: + inputs = F.Dropout(inputs, p=drop_inputs, axes=(axis,)) + + if valid_length is None: + def loop_body(inputs, states): + return cell(inputs, states) + else: + zeros = [] + for s in states: + zeros.append(F.zeros_like(s)) + states = list(_as_list(states)) + states.append(F.zeros((1))) + def loop_body(inputs, states): + cell_states = states[:-1] + iter_no = states[-1] + out, new_states = cell(inputs, cell_states) + for i, state in enumerate(cell_states): + new_states[i] = F.where(F.broadcast_greater(valid_length, iter_no), + new_states[i], state) + new_states.append(iter_no + 1) + return out, new_states + + outputs, states = F.contrib.foreach(loop_body, inputs, states) + if drop_outputs: + outputs = F.Dropout(outputs, p=drop_outputs, axes=(axis,)) + if valid_length is not None: + if axis != 0: + outputs = F.transpose(outputs, axes) + outputs = F.SequenceMask(outputs, sequence_length=valid_length, + use_sequence_length=True, axis=axis) + # the last state is the iteration number. We don't need it. + return outputs, states[:-1] + else: + if axis != 0: + outputs = F.transpose(outputs, axes) + return outputs, states diff --git a/tests/python/unittest/test_gluon_contrib.py b/tests/python/unittest/test_gluon_contrib.py index 6901e8bd12fe..1e0555900f17 100644 --- a/tests/python/unittest/test_gluon_contrib.py +++ b/tests/python/unittest/test_gluon_contrib.py @@ -17,12 +17,14 @@ from __future__ import print_function import mxnet as mx +import copy +from mxnet import gluon from mxnet.gluon import contrib from mxnet.gluon import nn from mxnet.gluon.contrib.nn import ( Concurrent, HybridConcurrent, Identity, SparseEmbedding, PixelShuffle1D, PixelShuffle2D, PixelShuffle3D) -from mxnet.test_utils import almost_equal +from mxnet.test_utils import almost_equal, default_context, assert_almost_equal from common import setup_module, with_seed, teardown import numpy as np from numpy.testing import assert_allclose @@ -313,6 +315,97 @@ def test_sampler(): assert list(interval_sampler) == [0, 3, 6, 9] +class TestRNNLayer(gluon.HybridBlock): + def __init__(self, cell_type, hidden_size, layout, prefix=None, params=None): + super(TestRNNLayer, self).__init__(prefix=prefix, params=params) + self.cell = cell_type(hidden_size, prefix='rnn_') + self.layout = layout + + def hybrid_forward(self, F, inputs, states, valid_length): + if isinstance(valid_length, list) and len(valid_length) == 0: + valid_length = None + return contrib.rnn.rnn_cell.dynamic_unroll(self.cell, inputs, states, + valid_length=valid_length, + layout=self.layout) + +def check_unroll(cell_type, num_states, layout): + batch_size = 20 + input_size = 50 + hidden_size = 30 + seq_len = 10 + if layout == 'TNC': + rnn_data = mx.nd.normal(loc=0, scale=1, shape=(seq_len, batch_size, input_size)) + elif layout == 'NTC': + rnn_data = mx.nd.normal(loc=0, scale=1, shape=(batch_size, seq_len, input_size)) + else: + print("Wrong layout") + return + valid_length = mx.nd.round(mx.nd.random.uniform(low=1, high=10, shape=(batch_size))) + state_shape = (batch_size, hidden_size) + states = [mx.nd.normal(loc=0, scale=1, shape=state_shape) for i in range(num_states)] + + cell = cell_type(hidden_size, prefix='rnn_') + cell.initialize(ctx=default_context()) + if layout == 'TNC': + cell(rnn_data[0], states) + else: + cell(rnn_data[:,0,:], states) + params1 = cell.collect_params() + orig_params1 = copy.deepcopy(params1) + + trainer = gluon.Trainer(params1, 'sgd', {'learning_rate' : 0.03}) + with mx.autograd.record(): + res1, states1 = cell.unroll(seq_len, rnn_data, states, valid_length=valid_length, + layout=layout, merge_outputs=True) + res1.backward() + trainer.step(batch_size) + + configs = [ + lambda layer: None, + lambda layer: layer.hybridize(), + lambda layer: layer.hybridize({'inline_limit': 0}), + lambda layer: layer.hybridize({'static_alloc': True}), + lambda layer: layer.hybridize({'static_alloc': True, 'static_shape': True}) ] + # We can't pass None to a hybrid block, but it accepts an empty list. + # so we use an empty list to represent valid_length if it's None. + if valid_length is None: + valid_length = [] + for config in configs: + layer = TestRNNLayer(cell_type, hidden_size, layout) + layer.initialize(ctx=default_context()) + config(layer) + res2, states2 = layer(rnn_data, states, valid_length) + params2 = layer.collect_params() + for key, val in orig_params1.items(): + params2[key].set_data(copy.deepcopy(val.data())) + + trainer = gluon.Trainer(params2, 'sgd', {'learning_rate' : 0.03}) + with mx.autograd.record(): + res2, states2 = layer(rnn_data, states, valid_length) + assert_almost_equal(res1.asnumpy(), res2.asnumpy(), rtol=0.001, atol=0.0001) + assert len(states1) == len(states2) + for i in range(len(states1)): + assert_almost_equal(states1[i].asnumpy(), states2[i].asnumpy(), + rtol=0.001, atol=0.0001) + res2.backward() + trainer.step(batch_size) + + for key, val in params1.items(): + weight1 = val.data() + weight2 = params2[key].data() + assert_almost_equal(weight1.asnumpy(), weight2.asnumpy(), + rtol=0.001, atol=0.0001) + + +@with_seed() +def test_contrib_unroll(): + cell_types = [(gluon.rnn.RNNCell, 1), (gluon.rnn.LSTMCell, 2), + (gluon.rnn.GRUCell, 1)] + for cell_type, num_states in cell_types: + check_unroll(cell_type, num_states, 'TNC') + check_unroll(cell_type, num_states, 'NTC') + + if __name__ == '__main__': import nose nose.runmodule() From d2cc39639d8f4da9c2e7a7d7cb0dd0b2c43599aa Mon Sep 17 00:00:00 2001 From: Tao Lv Date: Sat, 16 Feb 2019 12:23:30 +0800 Subject: [PATCH 35/75] [v1.4.x] Update MKL-DNN to fix the OSX build issue (#14141) (#14182) * update mkldnn to 0.17.x * update mkldnn to 0.17.4 * empty commit --- 3rdparty/mkldnn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/mkldnn b/3rdparty/mkldnn index a7c5f53832ac..722901c9aaef 160000 --- a/3rdparty/mkldnn +++ b/3rdparty/mkldnn @@ -1 +1 @@ -Subproject commit a7c5f53832acabade6e5086e72c960adedb3c38a +Subproject commit 722901c9aaefa579698df778d061d4848ab8c3e3 From 29fa446915a0dedfedf03226a5795f9a903d2b4f Mon Sep 17 00:00:00 2001 From: Ming Date: Sat, 16 Feb 2019 12:34:26 +0800 Subject: [PATCH 36/75] Fix broken amalgamation (#12792) * Fix broken amalgamation * Fixes #8850 * Update CONTRIBUTORS.md * Update CONTRIBUTORS.md --- CONTRIBUTORS.md | 2 ++ amalgamation/amalgamation.py | 7 ++++++- amalgamation/dmlc-minimum0.cc | 1 + amalgamation/mxnet_predict0.cc | 7 +++++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index ffe0da657aae..caec6cfacdf7 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -213,6 +213,8 @@ List of Contributors * [Piyush Ghai](https://github.com/piyushghai) * [Zach Boldyga](https://github.com/zboldyga) +* [Ming Yang](http://ufoym.com) + Label Bot --------- * [mxnet-label-bot](https://github.com/mxnet-label-bot) diff --git a/amalgamation/amalgamation.py b/amalgamation/amalgamation.py index a3c28f7118e9..0a4be02b8ff9 100644 --- a/amalgamation/amalgamation.py +++ b/amalgamation/amalgamation.py @@ -143,7 +143,12 @@ def expand(x, pending, stage): continue path = m.groups()[0] h = path.strip('./') if "../3rdparty/" not in path else path - source = find_source(h, x, stage) + if h.endswith('complex.h') and x.endswith('openblas_config.h'): + source = '' + elif h.startswith('ps/'): + source = '../3rdparty/ps-lite/include/' + h + else: + source = find_source(h, x, stage) if not source: if (h not in blacklist and h not in sysheaders and diff --git a/amalgamation/dmlc-minimum0.cc b/amalgamation/dmlc-minimum0.cc index 87e08d31c4d6..2fe629b20ed3 100644 --- a/amalgamation/dmlc-minimum0.cc +++ b/amalgamation/dmlc-minimum0.cc @@ -29,6 +29,7 @@ #include "../3rdparty/dmlc-core/src/io/local_filesys.cc" #include "../3rdparty/dmlc-core/src/data.cc" #include "../3rdparty/dmlc-core/src/io.cc" +#include "../3rdparty/dmlc-core/src/io/filesys.cc" #include "../3rdparty/dmlc-core/src/recordio.cc" diff --git a/amalgamation/mxnet_predict0.cc b/amalgamation/mxnet_predict0.cc index c4653d30dd3d..a18e28fd57de 100644 --- a/amalgamation/mxnet_predict0.cc +++ b/amalgamation/mxnet_predict0.cc @@ -43,25 +43,31 @@ #define DISABLE_OPENMP 1 #define DMLC_LOG_STACK_TRACE 0 +#include "src/common/utils.cc" #include "src/ndarray/ndarray_function.cc" #include "src/ndarray/ndarray.cc" #include "src/imperative/imperative.cc" +#include "src/imperative/imperative_utils.cc" #include "src/imperative/cached_op.cc" #include "src/engine/engine.cc" #include "src/engine/naive_engine.cc" +#include "src/engine/openmp.cc" #include "src/profiler/profiler.cc" +#include "src/profiler/aggregate_stats.cc" #include "src/executor/graph_executor.cc" #include "src/executor/attach_op_execs_pass.cc" #include "src/executor/attach_op_resource_pass.cc" #include "src/executor/inplace_addto_detect_pass.cc" +#include "src/executor/infer_graph_attr_pass.cc" #include "src/nnvm/legacy_json_util.cc" #include "src/nnvm/legacy_op_util.cc" +#include "src/nnvm/graph_editor.cc" #include "src/operator/operator.cc" #include "src/operator/operator_util.cc" @@ -93,4 +99,5 @@ #include "src/c_api/c_api_symbolic.cc" #include "src/c_api/c_api_ndarray.cc" #include "src/c_api/c_api_error.cc" +#include "src/c_api/c_api_profile.cc" From ed0b791ed5c2c2dde8b7136302abbbd3a85e89da Mon Sep 17 00:00:00 2001 From: Chaitanya Prakash Bapat Date: Sat, 16 Feb 2019 00:01:30 -0500 Subject: [PATCH 37/75] Fix nd.pick large array issue (#14082) * large op support * replaced size_t with index_t for M, added test case * changed shape --- src/operator/tensor/broadcast_reduce_op.h | 12 ++++++------ tests/nightly/test_large_array.py | 5 +++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/operator/tensor/broadcast_reduce_op.h b/src/operator/tensor/broadcast_reduce_op.h index 1edcb5a74a77..6aeeadfe820d 100644 --- a/src/operator/tensor/broadcast_reduce_op.h +++ b/src/operator/tensor/broadcast_reduce_op.h @@ -1172,12 +1172,12 @@ void L2NormComputeEx(const nnvm::NodeAttrs& attrs, template struct pick { template - MSHADOW_XINLINE static void Map(int i, DType* out, const DType* a, - const IType *idx, int M, int stride, + MSHADOW_XINLINE static void Map(index_t i, DType* out, const DType* a, + const IType *idx, index_t M, int stride, mshadow::Shape bshape, mshadow::Shape sshape) { using namespace broadcast; - int j = static_cast(idx[i]); + index_t j = static_cast(idx[i]); if (clip) { if (j <= 0) j = 0; else if (j >= M) j = M - 1; @@ -1194,12 +1194,12 @@ struct pick { template struct pick_grad { template - MSHADOW_XINLINE static void Map(int i, DType* igrad, const DType* ograd, - const IType *idx, int M, int stride, + MSHADOW_XINLINE static void Map(index_t i, DType* igrad, const DType* ograd, + const IType *idx, index_t M, int stride, mshadow::Shape bshape, mshadow::Shape sshape) { using namespace broadcast; - int j = static_cast(idx[i]); + index_t j = static_cast(idx[i]); if (clip) { if (j <= 0) j = 0; else if (j >= M) j = M - 1; diff --git a/tests/nightly/test_large_array.py b/tests/nightly/test_large_array.py index 696fdb1d4175..0249f44932c2 100644 --- a/tests/nightly/test_large_array.py +++ b/tests/nightly/test_large_array.py @@ -145,6 +145,11 @@ def test_where(): res = nd.sparse.where(csr_cond, a, b) assert np.sum(res[0].asnumpy() == 1) == b.shape[1] +def test_pick(): + a = mx.nd.ones(shape=(256*35, 1024*1024)) + b = mx.nd.ones(shape=(256*35,)) + res = mx.nd.pick(a,b) + assert res.shape == b.shape if __name__ == '__main__': import nose From 8bfbb7de46ce309e1935967ea2dfeb99f8d8a1f0 Mon Sep 17 00:00:00 2001 From: Shufan <33112206+juliusshufan@users.noreply.github.com> Date: Sat, 16 Feb 2019 19:06:25 +0800 Subject: [PATCH 38/75] Add an inference script providing both accuracy and benchmark result for original wide_n_deep example (#13895) * Add a inference script can provide both accuracy and benchmark result * minor changes * minor fix to use keep similar coding style as other examples * fix typo * remove code redundance and other minor changes * Addressing review comments and minor pylint fix * remove parameter 'accuracy' to make logic simple --- example/sparse/wide_deep/README.md | 5 +- example/sparse/wide_deep/config.py | 28 +++++++ example/sparse/wide_deep/inference.py | 106 ++++++++++++++++++++++++++ example/sparse/wide_deep/train.py | 20 +---- 4 files changed, 142 insertions(+), 17 deletions(-) create mode 100644 example/sparse/wide_deep/config.py create mode 100644 example/sparse/wide_deep/inference.py diff --git a/example/sparse/wide_deep/README.md b/example/sparse/wide_deep/README.md index 769d72365402..d0ae8ad65825 100644 --- a/example/sparse/wide_deep/README.md +++ b/example/sparse/wide_deep/README.md @@ -20,5 +20,8 @@ The example demonstrates how to train [wide and deep model](https://arxiv.org/abs/1606.07792). The [Census Income Data Set](https://archive.ics.uci.edu/ml/datasets/Census+Income) that this example uses for training is hosted by the [UC Irvine Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/). Tricks of feature engineering are adapted from tensorflow's [wide and deep tutorial](https://github.com/tensorflow/models/tree/master/official/wide_deep). The final accuracy should be around 85%. - +For training: - `python train.py` + +For inference: +- `python inference.py` diff --git a/example/sparse/wide_deep/config.py b/example/sparse/wide_deep/config.py new file mode 100644 index 000000000000..c0d20c49f981 --- /dev/null +++ b/example/sparse/wide_deep/config.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Related to feature engineering, please see preprocess in data.py +ADULT = { + 'train': 'adult.data', + 'test': 'adult.test', + 'url': 'https://archive.ics.uci.edu/ml/machine-learning-databases/adult/', + 'num_linear_features': 3000, + 'num_embed_features': 2, + 'num_cont_features': 38, + 'embed_input_dims': [1000, 1000], + 'hidden_units': [8, 50, 100], +} diff --git a/example/sparse/wide_deep/inference.py b/example/sparse/wide_deep/inference.py new file mode 100644 index 000000000000..e14396e50c15 --- /dev/null +++ b/example/sparse/wide_deep/inference.py @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +from mxnet.test_utils import * +from config import * +from data import get_uci_adult +from model import wide_deep_model +import argparse +import os +import time + +parser = argparse.ArgumentParser(description="Run sparse wide and deep inference", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--num-infer-batch', type=int, default=100, + help='number of batches to inference') +parser.add_argument('--load-epoch', type=int, default=0, + help='loading the params of the corresponding training epoch.') +parser.add_argument('--batch-size', type=int, default=100, + help='number of examples per batch') +parser.add_argument('--benchmark', action='store_true', default=False, + help='run the script for benchmark mode, not set for accuracy test.') +parser.add_argument('--verbose', action='store_true', default=False, + help='accurcy for each batch will be logged if set') +parser.add_argument('--gpu', action='store_true', default=False, + help='Inference on GPU with CUDA') +parser.add_argument('--model-prefix', type=str, default='checkpoint', + help='the model prefix') + +if __name__ == '__main__': + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.INFO, format=head) + + # arg parser + args = parser.parse_args() + logging.info(args) + num_iters = args.num_infer_batch + batch_size = args.batch_size + benchmark = args.benchmark + verbose = args.verbose + model_prefix = args.model_prefix + load_epoch = args.load_epoch + ctx = mx.gpu(0) if args.gpu else mx.cpu() + # dataset + data_dir = os.path.join(os.getcwd(), 'data') + val_data = os.path.join(data_dir, ADULT['test']) + val_csr, val_dns, val_label = get_uci_adult(data_dir, ADULT['test'], ADULT['url']) + # load parameters and symbol + sym, arg_params, aux_params = mx.model.load_checkpoint(model_prefix, load_epoch) + # data iterator + eval_data = mx.io.NDArrayIter({'csr_data': val_csr, 'dns_data': val_dns}, + {'softmax_label': val_label}, batch_size, + shuffle=True, last_batch_handle='discard') + # module + mod = mx.mod.Module(symbol=sym, context=ctx, data_names=['csr_data', 'dns_data'], + label_names=['softmax_label']) + mod.bind(data_shapes=eval_data.provide_data, label_shapes=eval_data.provide_label) + # get the sparse weight parameter + mod.set_params(arg_params=arg_params, aux_params=aux_params) + + data_iter = iter(eval_data) + nbatch = 0 + if benchmark: + logging.info('Inference benchmark started ...') + tic = time.time() + for i in range(num_iters): + try: + batch = data_iter.next() + except StopIteration: + data_iter.reset() + else: + mod.forward(batch, is_train=False) + for output in mod.get_outputs(): + output.wait_to_read() + nbatch += 1 + score = (nbatch*batch_size)/(time.time() - tic) + logging.info('batch size %d, process %s samples/s' % (batch_size, score)) + else: + logging.info('Inference started ...') + # use accuracy as the metric + metric = mx.metric.create(['acc']) + accuracy_avg = 0.0 + for batch in data_iter: + nbatch += 1 + metric.reset() + mod.forward(batch, is_train=False) + mod.update_metric(metric, batch.label) + accuracy_avg += metric.get()[1][0] + if args.verbose: + logging.info('batch %d, accuracy = %s' % (nbatch, metric.get())) + logging.info('averged accuracy on eval set is %.5f' % (accuracy_avg/nbatch)) diff --git a/example/sparse/wide_deep/train.py b/example/sparse/wide_deep/train.py index 6fd81b7fa480..eea70301660d 100644 --- a/example/sparse/wide_deep/train.py +++ b/example/sparse/wide_deep/train.py @@ -17,6 +17,7 @@ import mxnet as mx from mxnet.test_utils import * +from config import * from data import get_uci_adult from model import wide_deep_model import argparse @@ -31,7 +32,7 @@ help='number of examples per batch') parser.add_argument('--lr', type=float, default=0.001, help='learning rate') -parser.add_argument('--cuda', action='store_true', default=False, +parser.add_argument('--gpu', action='store_true', default=False, help='Train on GPU with CUDA') parser.add_argument('--optimizer', type=str, default='adam', help='what optimizer to use', @@ -40,19 +41,6 @@ help='number of batches to wait before logging training status') -# Related to feature engineering, please see preprocess in data.py -ADULT = { - 'train': 'adult.data', - 'test': 'adult.test', - 'url': 'https://archive.ics.uci.edu/ml/machine-learning-databases/adult/', - 'num_linear_features': 3000, - 'num_embed_features': 2, - 'num_cont_features': 38, - 'embed_input_dims': [1000, 1000], - 'hidden_units': [8, 50, 100], -} - - if __name__ == '__main__': import logging head = '%(asctime)-15s %(message)s' @@ -66,7 +54,7 @@ optimizer = args.optimizer log_interval = args.log_interval lr = args.lr - ctx = mx.gpu(0) if args.cuda else mx.cpu() + ctx = mx.gpu(0) if args.gpu else mx.cpu() # dataset data_dir = os.path.join(os.getcwd(), 'data') @@ -88,7 +76,7 @@ shuffle=True, last_batch_handle='discard') # module - mod = mx.mod.Module(symbol=model, context=ctx ,data_names=['csr_data', 'dns_data'], + mod = mx.mod.Module(symbol=model, context=ctx, data_names=['csr_data', 'dns_data'], label_names=['softmax_label']) mod.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label) mod.init_params() From 059f49f5d3c410252cd222f912c36a31fdcbfe3d Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Sat, 16 Feb 2019 20:15:15 +0100 Subject: [PATCH 39/75] Added USE_SIGNAL_HANDLER to other Linux builds which didn't had it (#14122) --- ci/docker/runtime_functions.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 4d9b256634a6..1b4caa073570 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -292,6 +292,7 @@ build_centos7_cpu() { USE_BLAS=openblas \ USE_MKLDNN=0 \ USE_DIST_KVSTORE=1 \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -326,6 +327,7 @@ build_centos7_mkldnn() { USE_LAPACK=1 \ USE_LAPACK_PATH=/usr/lib64/liblapack.so \ USE_BLAS=openblas \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -366,6 +368,7 @@ build_ubuntu_cpu_openblas() { USE_MKLDNN=0 \ USE_DIST_KVSTORE=1 \ USE_LIBJPEG_TURBO=1 \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -381,6 +384,7 @@ build_ubuntu_cpu_mkl() { USE_MKLDNN=0 \ USE_INTEL_PATH=/opt/intel \ USE_DIST_KVSTORE=1 \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -511,6 +515,7 @@ build_ubuntu_cpu_clang39_mkldnn() { USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ USE_OPENMP=0 \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -527,6 +532,7 @@ build_ubuntu_cpu_clang60_mkldnn() { USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ USE_OPENMP=1 \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -540,6 +546,7 @@ build_ubuntu_cpu_mkldnn() { ENABLE_TESTCOVERAGE=1 \ USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -553,6 +560,7 @@ build_ubuntu_cpu_mkldnn_mkl() { ENABLE_TESTCOVERAGE=1 \ USE_CPP_PACKAGE=1 \ USE_BLAS=mkl \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -636,6 +644,7 @@ build_ubuntu_gpu_mkldnn() { USE_CUDA_PATH=/usr/local/cuda \ USE_CUDNN=1 \ CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -652,6 +661,7 @@ build_ubuntu_gpu_mkldnn_nocudnn() { USE_CUDA_PATH=/usr/local/cuda \ USE_CUDNN=0 \ CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -670,6 +680,7 @@ build_ubuntu_gpu_cuda91_cudnn7() { USE_CPP_PACKAGE=1 \ USE_DIST_KVSTORE=1 \ CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } From 5adb6fcfb3ea50c76d387e63ce367bbe8c3f18d3 Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Sat, 16 Feb 2019 15:17:33 -0800 Subject: [PATCH 40/75] Add NHWC layout support to Pooling (cpu, gpu cuda, gpu cuDNN) (#13749) * Adds layout support: mx.sym.Pooling(..., layout='NHWC',...) with tests. * Docs changes * Trigger * Skip NHWC pooling tests on non-cuDNN platforms * Fix pylint NHWC pooling * Fixes from review * Add CuDNNPoolingOp::Supports() in place of Forward()/Backward() bool return. * Add layout support to cpu implementation of Pooling, with tests. * Fix cpplint. * Fix bug in cpu nhwc impl. * Add MXNet CUDA pooling in NWC, NHWC and NDHWC. Turn on 3D cuDNN pooling. Tests. * Add PoolingParam::GetLayout() for better default layout handling. * Fix cpplint. * Throw exception for quantization pooling not NCHW. * Expand nhwc pooling test coverage. * SupportMKLDNNPooling() to examine layout param. * Compare 'std' and 'v1' pooling versions only when op definitions permit. * Add pooling test diagnostic output. * Fix syntax. * Fix pooling FInplaceOption so it can be shared by all implementations. * Add missing param definition. * Fix #if logic. * Temp switch to DickJC123/mshadow: shows effect of half round-to-nearest on cpu. * Move back to dmlc/mshadow.git, now with float->half rounding. * Avoid underflow of lp pooling calc for dtype=float16. * Remove redundant pooling test. * Minor variable naming fixes. * Modify FInplaceOption handling per reviewer comments. Expand testing. * Correct gluon Pooling layout param description. * Correct Symbol Pooling description. * Use 'CHECK(x)' rather than 'if (x) LOG(FATAL)'. * Empty commit to trigger CI. --- python/mxnet/gluon/nn/conv_layers.py | 93 +- src/operator/nn/cudnn/cudnn_pooling-inl.h | 239 +++- src/operator/nn/mkldnn/mkldnn_pooling-inl.h | 3 +- src/operator/nn/pool.cuh | 479 +++++--- src/operator/nn/pool.h | 1010 +++++++++++++++-- src/operator/nn/pool_utils.h | 18 +- src/operator/nn/pooling-inl.h | 73 +- src/operator/nn/pooling.cc | 207 ++-- src/operator/nn/pooling.cu | 28 +- .../quantization/quantized_pooling.cc | 3 + tests/python/gpu/test_operator_gpu.py | 460 ++++---- tests/python/unittest/test_gluon.py | 165 +-- 12 files changed, 2024 insertions(+), 754 deletions(-) diff --git a/python/mxnet/gluon/nn/conv_layers.py b/python/mxnet/gluon/nn/conv_layers.py index 5f20d20c02ab..c210081f6071 100644 --- a/python/mxnet/gluon/nn/conv_layers.py +++ b/python/mxnet/gluon/nn/conv_layers.py @@ -673,7 +673,7 @@ def __init__(self, channels, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), class _Pooling(HybridBlock): """Abstract class for different pooling layers.""" def __init__(self, pool_size, strides, padding, ceil_mode, global_pool, - pool_type, count_include_pad=None, **kwargs): + pool_type, layout, count_include_pad=None, **kwargs): super(_Pooling, self).__init__(**kwargs) if strides is None: strides = pool_size @@ -684,6 +684,7 @@ def __init__(self, pool_size, strides, padding, ceil_mode, global_pool, self._kwargs = { 'kernel': pool_size, 'stride': strides, 'pad': padding, 'global_pool': global_pool, 'pool_type': pool_type, + 'layout': layout, 'pooling_convention': 'full' if ceil_mode else 'valid'} if count_include_pad is not None: self._kwargs['count_include_pad'] = count_include_pad @@ -695,7 +696,8 @@ def hybrid_forward(self, F, x): return F.Pooling(x, name='fwd', **self._kwargs) def __repr__(self): - s = '{name}(size={kernel}, stride={stride}, padding={pad}, ceil_mode={ceil_mode})' + s = '{name}(size={kernel}, stride={stride}, padding={pad}, ceil_mode={ceil_mode}' + s += ', global_pool={global_pool}, pool_type={pool_type}, layout={layout})' return s.format(name=self.__class__.__name__, ceil_mode=self._kwargs['pooling_convention'] == 'full', **self._kwargs) @@ -716,7 +718,7 @@ class MaxPool1D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. layout : str, default 'NCW' - Dimension ordering of data and weight. Only supports 'NCW' layout for now. + Dimension ordering of data and out ('NCW' or 'NWC'). 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions respectively. Pooling is applied on the W dimension. ceil_mode : bool, default False @@ -738,12 +740,13 @@ class MaxPool1D(_Pooling): """ def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', ceil_mode=False, **kwargs): - assert layout == 'NCW', "Only supports 'NCW' layout for now" + assert layout in ('NCW', 'NWC'),\ + "Only NCW and NWC layouts are valid for 1D Pooling" if isinstance(pool_size, numeric_types): pool_size = (pool_size,) assert len(pool_size) == 1, "pool_size must be a number or a list of 1 ints" super(MaxPool1D, self).__init__( - pool_size, strides, padding, ceil_mode, False, 'max', **kwargs) + pool_size, strides, padding, ceil_mode, False, 'max', layout, **kwargs) class MaxPool2D(_Pooling): @@ -761,7 +764,7 @@ class MaxPool2D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. layout : str, default 'NCHW' - Dimension ordering of data and weight. Only supports 'NCHW' layout for now. + Dimension ordering of data and out ('NCHW' or 'NHWC'). 'N', 'C', 'H', 'W' stands for batch, channel, height, and width dimensions respectively. padding is applied on 'H' and 'W' dimension. ceil_mode : bool, default False @@ -786,12 +789,13 @@ class MaxPool2D(_Pooling): """ def __init__(self, pool_size=(2, 2), strides=None, padding=0, layout='NCHW', ceil_mode=False, **kwargs): - assert layout == 'NCHW', "Only supports 'NCHW' layout for now" + assert layout in ('NCHW', 'NHWC'),\ + "Only NCHW and NHWC layouts are valid for 2D Pooling" if isinstance(pool_size, numeric_types): pool_size = (pool_size,)*2 assert len(pool_size) == 2, "pool_size must be a number or a list of 2 ints" super(MaxPool2D, self).__init__( - pool_size, strides, padding, ceil_mode, False, 'max', **kwargs) + pool_size, strides, padding, ceil_mode, False, 'max', layout, **kwargs) class MaxPool3D(_Pooling): @@ -809,7 +813,7 @@ class MaxPool3D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. layout : str, default 'NCDHW' - Dimension ordering of data and weight. Only supports 'NCDHW' layout for now. + Dimension ordering of data and out ('NCDHW' or 'NDHWC'). 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and depth dimensions respectively. padding is applied on 'D', 'H' and 'W' dimension. @@ -836,12 +840,13 @@ class MaxPool3D(_Pooling): """ def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, ceil_mode=False, layout='NCDHW', **kwargs): - assert layout == 'NCDHW', "Only supports 'NCDHW' layout for now" + assert layout in ('NCDHW', 'NDHWC'),\ + "Only NCDHW and NDHWC layouts are valid for 3D Pooling" if isinstance(pool_size, numeric_types): pool_size = (pool_size,)*3 assert len(pool_size) == 3, "pool_size must be a number or a list of 3 ints" super(MaxPool3D, self).__init__( - pool_size, strides, padding, ceil_mode, False, 'max', **kwargs) + pool_size, strides, padding, ceil_mode, False, 'max', layout, **kwargs) class AvgPool1D(_Pooling): @@ -858,7 +863,7 @@ class AvgPool1D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. layout : str, default 'NCW' - Dimension ordering of data and weight. Only supports 'NCW' layout for now. + Dimension ordering of data and out ('NCW' or 'NWC'). 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions respectively. padding is applied on 'W' dimension. ceil_mode : bool, default False @@ -882,12 +887,14 @@ class AvgPool1D(_Pooling): """ def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', ceil_mode=False, count_include_pad=True, **kwargs): - assert layout == 'NCW', "Only supports 'NCW' layout for now" + assert layout in ('NCW', 'NWC'),\ + "Only NCW and NWC layouts are valid for 1D Pooling" if isinstance(pool_size, numeric_types): pool_size = (pool_size,) assert len(pool_size) == 1, "pool_size must be a number or a list of 1 ints" super(AvgPool1D, self).__init__( - pool_size, strides, padding, ceil_mode, False, 'avg', count_include_pad, **kwargs) + pool_size, strides, padding, ceil_mode, False, 'avg', layout, count_include_pad, + **kwargs) class AvgPool2D(_Pooling): @@ -904,7 +911,7 @@ class AvgPool2D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. layout : str, default 'NCHW' - Dimension ordering of data and weight. Only supports 'NCHW' layout for now. + Dimension ordering of data and out ('NCHW' or 'NHWC'). 'N', 'C', 'H', 'W' stands for batch, channel, height, and width dimensions respectively. padding is applied on 'H' and 'W' dimension. ceil_mode : bool, default False @@ -931,12 +938,14 @@ class AvgPool2D(_Pooling): """ def __init__(self, pool_size=(2, 2), strides=None, padding=0, ceil_mode=False, layout='NCHW', count_include_pad=True, **kwargs): - assert layout == 'NCHW', "Only supports 'NCHW' layout for now" + assert layout in ('NCHW', 'NHWC'),\ + "Only NCHW and NHWC layouts are valid for 2D Pooling" if isinstance(pool_size, numeric_types): pool_size = (pool_size,)*2 assert len(pool_size) == 2, "pool_size must be a number or a list of 2 ints" super(AvgPool2D, self).__init__( - pool_size, strides, padding, ceil_mode, False, 'avg', count_include_pad, **kwargs) + pool_size, strides, padding, ceil_mode, False, 'avg', layout, count_include_pad, + **kwargs) class AvgPool3D(_Pooling): @@ -953,7 +962,7 @@ class AvgPool3D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. layout : str, default 'NCDHW' - Dimension ordering of data and weight. Can be 'NCDHW', 'NDHWC', etc. + Dimension ordering of data and out ('NCDHW' or 'NDHWC'). 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and depth dimensions respectively. padding is applied on 'D', 'H' and 'W' dimension. @@ -982,12 +991,14 @@ class AvgPool3D(_Pooling): """ def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, ceil_mode=False, layout='NCDHW', count_include_pad=True, **kwargs): - assert layout == 'NCDHW', "Only supports 'NCDHW' layout for now" + assert layout in ('NCDHW', 'NDHWC'),\ + "Only NCDHW and NDHWC layouts are valid for 3D Pooling" if isinstance(pool_size, numeric_types): pool_size = (pool_size,)*3 assert len(pool_size) == 3, "pool_size must be a number or a list of 3 ints" super(AvgPool3D, self).__init__( - pool_size, strides, padding, ceil_mode, False, 'avg', count_include_pad, **kwargs) + pool_size, strides, padding, ceil_mode, False, 'avg', layout, count_include_pad, + **kwargs) class GlobalMaxPool1D(_Pooling): @@ -997,7 +1008,7 @@ class GlobalMaxPool1D(_Pooling): Parameters ---------- layout : str, default 'NCW' - Dimension ordering of data and weight. Only supports 'NCW' layout for now. + Dimension ordering of data and out ('NCW' or 'NWC'). 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions respectively. Pooling is applied on the W dimension. @@ -1011,9 +1022,10 @@ class GlobalMaxPool1D(_Pooling): when `layout` is `NCW`. """ def __init__(self, layout='NCW', **kwargs): - assert layout == 'NCW', "Only supports 'NCW' layout for now" + assert layout in ('NCW', 'NWC'),\ + "Only NCW and NWC layouts are valid for 1D Pooling" super(GlobalMaxPool1D, self).__init__( - (1,), None, 0, True, True, 'max', **kwargs) + (1,), None, 0, True, True, 'max', layout, **kwargs) class GlobalMaxPool2D(_Pooling): @@ -1023,7 +1035,7 @@ class GlobalMaxPool2D(_Pooling): Parameters ---------- layout : str, default 'NCHW' - Dimension ordering of data and weight. Only supports 'NCHW' layout for now. + Dimension ordering of data and out ('NCHW' or 'NHWC'). 'N', 'C', 'H', 'W' stands for batch, channel, height, and width dimensions respectively. padding is applied on 'H' and 'W' dimension. @@ -1038,9 +1050,10 @@ class GlobalMaxPool2D(_Pooling): `(batch_size, channels, 1, 1)` when `layout` is `NCHW`. """ def __init__(self, layout='NCHW', **kwargs): - assert layout == 'NCHW', "Only supports 'NCHW' layout for now" + assert layout in ('NCHW', 'NHWC'),\ + "Only NCHW and NHWC layouts are valid for 2D Pooling" super(GlobalMaxPool2D, self).__init__( - (1, 1), None, 0, True, True, 'max', **kwargs) + (1, 1), None, 0, True, True, 'max', layout, **kwargs) class GlobalMaxPool3D(_Pooling): @@ -1050,7 +1063,7 @@ class GlobalMaxPool3D(_Pooling): Parameters ---------- layout : str, default 'NCDHW' - Dimension ordering of data and weight. Only supports 'NCDHW' layout for now. + Dimension ordering of data and out ('NCDHW' or 'NDHWC'). 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and depth dimensions respectively. padding is applied on 'D', 'H' and 'W' dimension. @@ -1066,9 +1079,10 @@ class GlobalMaxPool3D(_Pooling): `(batch_size, channels, 1, 1, 1)` when `layout` is `NCDHW`. """ def __init__(self, layout='NCDHW', **kwargs): - assert layout == 'NCDHW', "Only supports 'NCDHW' layout for now" + assert layout in ('NCDHW', 'NDHWC'),\ + "Only NCDHW and NDHWC layouts are valid for 3D Pooling" super(GlobalMaxPool3D, self).__init__( - (1, 1, 1), None, 0, True, True, 'max', **kwargs) + (1, 1, 1), None, 0, True, True, 'max', layout, **kwargs) class GlobalAvgPool1D(_Pooling): @@ -1077,7 +1091,7 @@ class GlobalAvgPool1D(_Pooling): Parameters ---------- layout : str, default 'NCW' - Dimension ordering of data and weight. Only supports 'NCW' layout for now. + Dimension ordering of data and out ('NCW' or 'NWC'). 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions respectively. padding is applied on 'W' dimension. @@ -1090,9 +1104,10 @@ class GlobalAvgPool1D(_Pooling): - **out**: 3D output tensor with shape `(batch_size, channels, 1)`. """ def __init__(self, layout='NCW', **kwargs): - assert layout == 'NCW', "Only supports 'NCW' layout for now" + assert layout in ('NCW', 'NWC'),\ + "Only NCW and NWC layouts are valid for 1D Pooling" super(GlobalAvgPool1D, self).__init__( - (1,), None, 0, True, True, 'avg', **kwargs) + (1,), None, 0, True, True, 'avg', layout, **kwargs) class GlobalAvgPool2D(_Pooling): @@ -1101,7 +1116,7 @@ class GlobalAvgPool2D(_Pooling): Parameters ---------- layout : str, default 'NCHW' - Dimension ordering of data and weight. Only supports 'NCHW' layout for now. + Dimension ordering of data and out ('NCHW' or 'NHWC'). 'N', 'C', 'H', 'W' stands for batch, channel, height, and width dimensions respectively. @@ -1116,9 +1131,10 @@ class GlobalAvgPool2D(_Pooling): `(batch_size, channels, 1, 1)` when `layout` is `NCHW`. """ def __init__(self, layout='NCHW', **kwargs): - assert layout == 'NCHW', "Only supports 'NCHW' layout for now" + assert layout in ('NCHW', 'NHWC'),\ + "Only NCHW and NHWC layouts are valid for 2D Pooling" super(GlobalAvgPool2D, self).__init__( - (1, 1), None, 0, True, True, 'avg', **kwargs) + (1, 1), None, 0, True, True, 'avg', layout, **kwargs) class GlobalAvgPool3D(_Pooling): @@ -1127,7 +1143,7 @@ class GlobalAvgPool3D(_Pooling): Parameters ---------- layout : str, default 'NCDHW' - Dimension ordering of data and weight. Can be 'NCDHW', 'NDHWC', etc. + Dimension ordering of data and out ('NCDHW' or 'NDHWC'). 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and depth dimensions respectively. padding is applied on 'D', 'H' and 'W' dimension. @@ -1143,9 +1159,10 @@ class GlobalAvgPool3D(_Pooling): `(batch_size, channels, 1, 1, 1)` when `layout` is `NCDHW`. """ def __init__(self, layout='NCDHW', **kwargs): - assert layout == 'NCDHW', "Only supports 'NCDHW' layout for now" + assert layout in ('NCDHW', 'NDHWC'),\ + "Only NCDHW and NDHWC layouts are valid for 3D Pooling" super(GlobalAvgPool3D, self).__init__( - (1, 1, 1), None, 0, True, True, 'avg', **kwargs) + (1, 1, 1), None, 0, True, True, 'avg', layout, **kwargs) class ReflectionPad2D(HybridBlock): diff --git a/src/operator/nn/cudnn/cudnn_pooling-inl.h b/src/operator/nn/cudnn/cudnn_pooling-inl.h index 89fa73ef5471..ada605db0ee9 100644 --- a/src/operator/nn/cudnn/cudnn_pooling-inl.h +++ b/src/operator/nn/cudnn/cudnn_pooling-inl.h @@ -21,13 +21,13 @@ * Copyright (c) 2015 by Contributors * \file cudnn_pooling-inl.h * \brief - * \author Bing Xu + * \author Bing Xu, Dick Carter */ #ifndef MXNET_OPERATOR_NN_CUDNN_CUDNN_POOLING_INL_H_ #define MXNET_OPERATOR_NN_CUDNN_CUDNN_POOLING_INL_H_ #include -#include +#include #include "../pooling-inl.h" namespace mxnet { @@ -63,7 +63,7 @@ class CuDNNPoolingOp { } break; default: - LOG(FATAL) << "Not implmented"; + LOG(FATAL) << "Pooling type not implemented by cuDNN."; } } @@ -81,7 +81,7 @@ class CuDNNPoolingOp { CHECK_EQ(s->dnn_handle_ownership_, mshadow::Stream::OwnHandle); typename DataType::ScaleType alpha = 1.0f; typename DataType::ScaleType beta = 0.0f; - this->Init(s, in_data, out_data); + CHECK(this->Init(s, in_data, out_data)) << "cuDNN Pooling invoked with unsupported parameters."; if (param_.kernel.ndim() == 2) { // 2d pool Tensor data = in_data.get(s); @@ -111,7 +111,7 @@ class CuDNNPoolingOp { out_desc_, out.dptr_)); } else { - LOG(FATAL) << "Only support 2D or 3D pooling"; + LOG(FATAL) << "cuDNN only supports 2D or 3D pooling."; } } @@ -125,7 +125,7 @@ class CuDNNPoolingOp { CHECK_EQ(s->dnn_handle_ownership_, mshadow::Stream::OwnHandle); typename DataType::ScaleType alpha = 1.0f; typename DataType::ScaleType beta = 0.0f; - this->Init(s, in_data, out_data); + CHECK(this->Init(s, in_data, out_data)) << "cuDNN Pooling invoked with unsupported parameters."; if (param_.kernel.ndim() == 2) { // 2d pool Tensor m_out_grad = out_grad.get(s); @@ -163,106 +163,219 @@ class CuDNNPoolingOp { in_desc_, m_in_grad.dptr_)); } else { - LOG(FATAL) << "Only support 2D or 3D pooling"; + LOG(FATAL) << "cuDNN only supports 2D or 3D pooling."; } } +/*! + * \brief Returns whether the cuDNN library version supports the pooling operation + * described by `param`: cuDNN v5 and earlier does not support 3D pooling for example. + * CuDNN v7.1.4 backprop kernel doesn't support kernel sizes 9 and above. + */ + static bool Supports(const PoolingParam ¶m, const TBlob& input) { + using namespace mshadow; + static bool sum_pooling_warning_issued = false; + static bool lp_pooling_warning_issued = false; + static bool unsupported_dim_warning_issued = false; + int layout = param.GetLayout(input.ndim()); + + switch (param.pool_type) { + case pool_enum::kMaxPooling: + case pool_enum::kAvgPooling: + break; + case pool_enum::kSumPooling: + if (!sum_pooling_warning_issued) { + sum_pooling_warning_issued = true; + LOG(WARNING) << "Sum pooling is not supported by cudnn, MXNet sum pooling is applied."; + } + return false; + case pool_enum::kLpPooling: + if (!lp_pooling_warning_issued) { + lp_pooling_warning_issued = true; + LOG(WARNING) << "Lp pooling is not supported by cudnn, MXNet Lp pooling is applied."; + } + return false; + default: + return false; + } + + if (param.kernel.ndim() == 2) { + // 2d pooling + if (!(layout == mshadow::kNCHW || layout == mshadow::kNHWC)) + return false; +#if CUDNN_VERSION == 7104 + // CuDNN v7.1.4 backprop kernel doesn't support kernel sizes 9 and above. + // Perform shape calculations in a standard (NCHW) layout space + mshadow::Shape<4> input_shape = input.shape_.get<4>(); + mshadow::Shape<4> dshape_nchw = (layout == mshadow::kNHWC) ? + ConvertLayout(input_shape, mshadow::kNHWC, mshadow::kNCHW) : + input_shape; + int kernel_height = param.global_pool ? dshape_nchw[2] : param.kernel[0]; + int kernel_width = param.global_pool ? dshape_nchw[3] : param.kernel[1]; + if (kernel_height > 8 || kernel_width > 8) + return false; +#endif +#if CUDNN_VERSION >= 7105 && CUDNN_VERSION < 7500 + // Avoid strided NHWC max pooling for some configs + if (layout == mshadow::kNHWC && + param.pool_type == pool_enum::kMaxPooling && !param.global_pool) { + if (param.stride[0] >= 3 || + param.stride[0] == 2 && param.kernel[0] % 2 == 0 && param.kernel[0] != 2) + return false; + if (param.stride[1] >= 3 || + param.stride[1] == 2 && param.kernel[1] % 2 == 0 && param.kernel[1] != 2) + return false; + } +#endif + } else if (param.kernel.ndim() == 3) { + // 3d pooling +#if CUDNN_MAJOR < 5 + LogUnsupportedDim(&unsupported_dim_warning_issued, param.kernel.ndim()); + return false; +#endif + if (!(layout == mshadow::kNCDHW || layout == mshadow::kNDHWC)) + return false; + } else { + // Unsupported kernel dim + LogUnsupportedDim(&unsupported_dim_warning_issued, param.kernel.ndim()); + return false; + } + + return true; + } + private: - inline void Init(mshadow::Stream *s, const TBlob &in_data, + // Return boolean saying whether pooling configuration is supported + inline bool Init(mshadow::Stream *s, const TBlob &in_data, const TBlob &out_data) { using namespace mshadow; + bool is_supported = true; #if CUDNN_MAJOR >= 5 nan_prop_ = CUDNN_NOT_PROPAGATE_NAN; #endif + int layout = param_.GetLayout(in_data.ndim()); if (param_.kernel.ndim() == 2) { - // 2d conv + // 2d pooling + CHECK(layout == mshadow::kNCHW || layout == mshadow::kNHWC) << "Need 2D layout NCHW or NHWC."; + cudnnTensorFormat_t cudnn_layout = (layout == mshadow::kNCHW) ? CUDNN_TENSOR_NCHW + : CUDNN_TENSOR_NHWC; Tensor data = in_data.get(s); Tensor out = out_data.get(s); - mshadow::Shape<4> dshape = data.shape_; + // Perform shape calculations in a standard (NCHW) layout space + mshadow::Shape<4> dshape_nchw = (layout == mshadow::kNHWC) ? + ConvertLayout(data.shape_, mshadow::kNHWC, mshadow::kNCHW) : + data.shape_; + mshadow::Shape<4> oshape_nchw = (layout == mshadow::kNHWC) ? + ConvertLayout(out.shape_, mshadow::kNHWC, mshadow::kNCHW) : + out.shape_; CUDNN_CALL(cudnnSetTensor4dDescriptor(in_desc_, - CUDNN_TENSOR_NCHW, + cudnn_layout, dtype_, - data.shape_[0], - data.shape_[1], - data.shape_[2], - data.shape_[3])); + dshape_nchw[0], + dshape_nchw[1], + dshape_nchw[2], + dshape_nchw[3])); CUDNN_CALL(cudnnSetTensor4dDescriptor(out_desc_, - CUDNN_TENSOR_NCHW, + cudnn_layout, dtype_, - out.shape_[0], - out.shape_[1], - out.shape_[2], - out.shape_[3])); + oshape_nchw[0], + oshape_nchw[1], + oshape_nchw[2], + oshape_nchw[3])); + int kernel_height = param_.global_pool ? dshape_nchw[2] : param_.kernel[0]; + int kernel_width = param_.global_pool ? dshape_nchw[3] : param_.kernel[1]; + // CuDNN v7.1.4 backprop kernel doesn't support kernel sizes 9 and above. + // For reference see Fixed Issues section in + // https://docs.nvidia.com/deeplearning/sdk/cudnn-release-notes/rel_721.html#rel_721 + #if CUDNN_VERSION == 7104 + is_supported = kernel_height <= 8 && kernel_width <= 8; + #endif #if CUDNN_MAJOR >= 5 CUDNN_CALL(cudnnSetPooling2dDescriptor(pooling_desc_, mode_, nan_prop_, - param_.global_pool ? dshape[2] : param_.kernel[0], - param_.global_pool ? dshape[3] : param_.kernel[1], + kernel_height, + kernel_width, param_.global_pool ? 0 : param_.pad[0], param_.global_pool ? 0 : param_.pad[1], param_.global_pool ? 1 : param_.stride[0], - param_.global_pool ? 1 :param_.stride[1])); + param_.global_pool ? 1 : param_.stride[1])); #else CUDNN_CALL(cudnnSetPooling2dDescriptor(pooling_desc_, mode_, - param_.global_pool ? dshape[2] : param_.kernel[0], - param_.global_pool ? dshape[3] : param_.kernel[1], + kernel_height, + kernel_width, param_.global_pool ? 0 : param_.pad[0], - param_.global_ppol ? 0 : param_.pad[1], + param_.global_pool ? 0 : param_.pad[1], param_.global_pool ? 1 : param_.stride[0], param_.global_pool ? 1 : param_.stride[1])); #endif } else { + CHECK(layout == mshadow::kNCDHW || + layout == mshadow::kNDHWC) << "Need 3D layout NCDHW or NDHWC."; Tensor data = in_data.get(s); - Tensor out = out_data.get(s); - std::vector ishape = {static_cast(data.shape_[0]), - static_cast(data.shape_[1]), - static_cast(data.shape_[2]), - static_cast(data.shape_[3]), - static_cast(data.shape_[4])}; - - std::vector istride = {static_cast(ishape[1] * ishape[2] * ishape[3] * ishape[4]), - static_cast(ishape[2] * ishape[3] * ishape[4]), - static_cast(ishape[3] * ishape[4]), - static_cast(ishape[4]), 1}; + mshadow::Shape<5> dshape = data.shape_; + mshadow::Shape<5> dstride = mshadow::Shape5(dshape.ProdShape(1, 5), + dshape.ProdShape(2, 5), + dshape.ProdShape(3, 5), + dshape.ProdShape(4, 5), + dshape.ProdShape(5, 5)); - std::vector oshape = {static_cast(out.shape_[0]), - static_cast(out.shape_[1]), - static_cast(out.shape_[2]), - static_cast(out.shape_[3]), - static_cast(out.shape_[4])}; + Tensor out = out_data.get(s); + mshadow::Shape<5> oshape = out.shape_; + mshadow::Shape<5> ostride = mshadow::Shape5(oshape.ProdShape(1, 5), + oshape.ProdShape(2, 5), + oshape.ProdShape(3, 5), + oshape.ProdShape(4, 5), + oshape.ProdShape(5, 5)); + // Convert to a standard (NCDHW) layout space to create args for cuDNN - std::vector ostride = {static_cast(oshape[1] * oshape[2] * oshape[3] * oshape[4]), - static_cast(oshape[2] * oshape[3] * oshape[4]), - static_cast(oshape[3] * oshape[4]), - static_cast(oshape[4]), 1}; + mshadow::Shape<5> dshape_ncdhw = (layout == mshadow::kNDHWC) ? + ConvertLayout(dshape, mshadow::kNDHWC, mshadow::kNCDHW) : + dshape; + mshadow::Shape<5> dstride_ncdhw = (layout == mshadow::kNDHWC) ? + ConvertLayout(dstride, mshadow::kNDHWC, mshadow::kNCDHW) : + dstride; + mshadow::Shape<5> oshape_ncdhw = (layout == mshadow::kNDHWC) ? + ConvertLayout(oshape, mshadow::kNDHWC, mshadow::kNCDHW) : + oshape; + mshadow::Shape<5> ostride_ncdhw = (layout == mshadow::kNDHWC) ? + ConvertLayout(ostride, mshadow::kNDHWC, mshadow::kNCDHW) : + ostride; + // Create int arrays for passing into cuDNN + std::array dshape_ncdhw_int, dstride_ncdhw_int, oshape_ncdhw_int, ostride_ncdhw_int; + for (int i = 0; i < 5; ++i) { + dshape_ncdhw_int[i] = static_cast(dshape_ncdhw[i]); + dstride_ncdhw_int[i] = static_cast(dstride_ncdhw[i]); + oshape_ncdhw_int[i] = static_cast(oshape_ncdhw[i]); + ostride_ncdhw_int[i] = static_cast(ostride_ncdhw[i]); + } - std::vector kernel_vec = {param_.global_pool ? ishape[2] : + std::array kernel_vec = {param_.global_pool ? static_cast(dshape_ncdhw[2]) : static_cast(param_.kernel[0]), - param_.global_pool ? ishape[3] : + param_.global_pool ? static_cast(dshape_ncdhw[3]) : static_cast(param_.kernel[1]), - param_.global_pool ? ishape[4] : + param_.global_pool ? static_cast(dshape_ncdhw[4]) : static_cast(param_.kernel[2])}; - std::vector pad_vec = {param_.global_pool ? 0 : static_cast(param_.pad[0]), + std::array pad_vec = {param_.global_pool ? 0 : static_cast(param_.pad[0]), param_.global_pool ? 0 : static_cast(param_.pad[1]), param_.global_pool ? 0 : static_cast(param_.pad[2])}; - std::vector stride_vec = {param_.global_pool ? 1 : static_cast(param_.stride[0]), + std::array stride_vec = {param_.global_pool ? 1 : static_cast(param_.stride[0]), param_.global_pool ? 1 : static_cast(param_.stride[1]), param_.global_pool ? 1 : static_cast(param_.stride[2])}; CUDNN_CALL(cudnnSetTensorNdDescriptor(in_desc_, dtype_, - static_cast(ishape.size()), - &ishape[0], - &istride[0])); + static_cast(dshape_ncdhw_int.size()), + &dshape_ncdhw_int[0], + &dstride_ncdhw_int[0])); CUDNN_CALL(cudnnSetTensorNdDescriptor(out_desc_, dtype_, - static_cast(oshape.size()), - &oshape[0], - &ostride[0])); + static_cast(oshape_ncdhw_int.size()), + &oshape_ncdhw_int[0], + &ostride_ncdhw_int[0])); #if CUDNN_MAJOR >= 5 CUDNN_CALL(cudnnSetPoolingNdDescriptor(pooling_desc_, mode_, @@ -272,9 +385,19 @@ class CuDNNPoolingOp { &(pad_vec[0]), &(stride_vec[0]))); #else - LOG(FATAL) << "3D pooling only support CUDNN v5 and above"; + LOG(FATAL) << "3D pooling is only supported by CUDNN v5 and above."; #endif } + return is_supported; + } + + // Log once that the dimension of the pooling operation isn't supported + static void LogUnsupportedDim(bool *msg_logged, int ndim) { + if (!*msg_logged) { + *msg_logged = true; + LOG(WARNING) << ndim << "D pooling is not supported by cudnn, " + << "MXNet " << ndim << "D pooling is applied."; + } } cudnnDataType_t dtype_; diff --git a/src/operator/nn/mkldnn/mkldnn_pooling-inl.h b/src/operator/nn/mkldnn/mkldnn_pooling-inl.h index f548778c7615..de3d63e24f6c 100644 --- a/src/operator/nn/mkldnn/mkldnn_pooling-inl.h +++ b/src/operator/nn/mkldnn/mkldnn_pooling-inl.h @@ -104,7 +104,8 @@ class MKLDNNPoolingBwd { inline bool SupportMKLDNNPooling(const PoolingParam ¶m) { return param.kernel.ndim() == 2 && (param.pool_type == pool_enum::kMaxPooling || - param.pool_type == pool_enum::kAvgPooling); + param.pool_type == pool_enum::kAvgPooling) && + (!param.layout.has_value() || param.layout.value() == mshadow::kNCHW); } inline bool SupportMKLDNNPooling(const PoolingParam ¶m, diff --git a/src/operator/nn/pool.cuh b/src/operator/nn/pool.cuh index 976aacf63a55..671bc7932ef9 100644 --- a/src/operator/nn/pool.cuh +++ b/src/operator/nn/pool.cuh @@ -89,29 +89,32 @@ namespace mxnet { namespace op { /*! - * \brief max pooling gpu kernel for 1-D images. + * \brief max pooling gpu kernel for 1-D images, for both NCW and NWC layouts. * Do not call this kernel directly. Use the interface pool(). */ -template +template __global__ void pool_max_1d_gpu_kernel(const int nthreads, const DType* in_data, const int channels, const int width, const int pooled_width, const int kernel_w, const int stride_w, const int pad_w, DType* out_data) { using mshadow::red::limits::MinValue; - // index is the output image's pixel index in NCW + // index is the output image's pixel index CUDA_KERNEL_LOOP(index, nthreads) { - const int pw = index % pooled_width; - const int c = (index / pooled_width) % channels; + const bool nwc_layout = layout == mshadow::kNWC; + const int idx = nwc_layout ? (index / channels) : index; + const int pw = idx % pooled_width; + const int c = nwc_layout ? (index % channels) : (index / pooled_width) % channels; const int n = index / pooled_width / channels; int wstart = pw * stride_w - pad_w; const int wend = min(wstart + kernel_w, width); wstart = max(wstart, 0); - const DType* in_slice = - in_data + (n * channels + c) * width; + const DType* in_slice = nwc_layout ? in_data + n * channels * width + c + : in_data + (n * channels + c) * width; DType max_val = MinValue(); + const int multiplier = nwc_layout ? channels : 1; for (int w = wstart; w < wend; ++w) { - const DType in_val = in_slice[w]; + const DType in_val = in_slice[w * multiplier]; if (in_val > max_val) { max_val = in_val; } @@ -121,10 +124,10 @@ __global__ void pool_max_1d_gpu_kernel(const int nthreads, const DType* in_data, } /*! - * \brief max pooling gpu kernel for 2-D images. + * \brief max pooling gpu kernel for 2-D images, for both NCHW and NHWC layouts. * Do not call this kernel directly. Use the interface pool(). */ -template +template __global__ void pool_max_2d_gpu_kernel(const int nthreads, const DType* in_data, const int channels, const int height, const int width, const int pooled_height, const int pooled_width, @@ -132,11 +135,14 @@ __global__ void pool_max_2d_gpu_kernel(const int nthreads, const DType* in_data, const int stride_w, const int pad_h, const int pad_w, DType* out_data) { using mshadow::red::limits::MinValue; - // index is the output image's pixel index in NCHW + // index is the output image's pixel index CUDA_KERNEL_LOOP(index, nthreads) { - const int pw = index % pooled_width; - const int ph = (index / pooled_width) % pooled_height; - const int c = (index / pooled_width / pooled_height) % channels; + const bool nhwc_layout = layout == mshadow::kNHWC; + const int idx = nhwc_layout ? (index / channels) : index; + const int pw = idx % pooled_width; + const int ph = (idx / pooled_width) % pooled_height; + const int c = nhwc_layout ? (index % channels) + : (index / pooled_width / pooled_height) % channels; const int n = index / pooled_width / pooled_height / channels; int hstart = ph * stride_h - pad_h; int wstart = pw * stride_w - pad_w; @@ -144,12 +150,13 @@ __global__ void pool_max_2d_gpu_kernel(const int nthreads, const DType* in_data, const int wend = min(wstart + kernel_w, width); hstart = max(hstart, 0); wstart = max(wstart, 0); - const DType* in_slice = - in_data + (n * channels + c) * height * width; + const DType* in_slice = nhwc_layout ? in_data + n * channels * height * width + c + : in_data + (n * channels + c) * height * width; DType max_val = MinValue(); + const int multiplier = nhwc_layout ? channels : 1; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - const DType in_val = in_slice[h * width + w]; + const DType in_val = in_slice[(h * width + w) * multiplier]; if (in_val > max_val) { max_val = in_val; } @@ -160,10 +167,10 @@ __global__ void pool_max_2d_gpu_kernel(const int nthreads, const DType* in_data, } /*! - * \brief max pooling gpu kernel for 3-D images. + * \brief max pooling gpu kernel for 3-D images, for both NCDHW and NDHWC layouts. * Do not call this kernel directly. Use the interface pool(). */ -template +template __global__ void pool_max_3d_gpu_kernel(const int nthreads, const DType* in_data, const int channels, const int depth, const int height, const int width, const int pooled_depth, const int pooled_height, @@ -173,12 +180,15 @@ __global__ void pool_max_3d_gpu_kernel(const int nthreads, const DType* in_data, const int pad_h, const int pad_w, DType* out_data) { using mshadow::red::limits::MinValue; - // index is the output image's pixel index in NCDHW + // index is the output image's pixel index CUDA_KERNEL_LOOP(index, nthreads) { - const int pw = index % pooled_width; - const int ph = (index / pooled_width) % pooled_height; - const int pd = (index / pooled_width / pooled_height) % pooled_depth; - const int c = (index / pooled_width / pooled_height / pooled_depth) % channels; + const bool ndhwc_layout = layout == mshadow::kNDHWC; + const int idx = ndhwc_layout ? (index / channels) : index; + const int pw = idx % pooled_width; + const int ph = (idx / pooled_width) % pooled_height; + const int pd = (idx / pooled_width / pooled_height) % pooled_depth; + const int c = ndhwc_layout ? (index % channels) + : (index / pooled_width / pooled_height / pooled_depth) % channels; const int n = index / pooled_width / pooled_height / pooled_depth / channels; int dstart = pd * stride_d - pad_d; int hstart = ph * stride_h - pad_h; @@ -189,13 +199,14 @@ __global__ void pool_max_3d_gpu_kernel(const int nthreads, const DType* in_data, dstart = max(dstart, 0); hstart = max(hstart, 0); wstart = max(wstart, 0); - const DType* in_slice = - in_data + (n * channels + c) * depth * height * width; + const DType* in_slice = ndhwc_layout ? in_data + n * channels * depth * height * width + c + : in_data + (n * channels + c) * depth * height * width; DType max_val = MinValue(); + const int multiplier = ndhwc_layout ? channels : 1; for (int d = dstart; d < dend; ++d) { for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - const DType in_val = in_slice[(d * height + h) * width + w]; + const DType in_val = in_slice[((d * height + h) * width + w) * multiplier]; if (in_val > max_val) { max_val = in_val; } @@ -207,17 +218,21 @@ __global__ void pool_max_3d_gpu_kernel(const int nthreads, const DType* in_data, } /*! - * \brief avg/sum pooling gpu kernel for 1-D images. + * \brief avg/sum pooling gpu kernel for 1-D images, for both NCW and NWC layouts. * Do not call this kernel directly. Use the interface pool(). */ -template +template __global__ void pool_sum_1d_gpu_kernel(const int nthreads, const DType* in_data, const int channels, const int width, const int pooled_width, const int kernel_w, const int stride_w, const int pad_w, DType* out_data, - const bool get_avg = false, const bool count_include_pad = true) { + const bool get_avg = false, + const bool count_include_pad = true) { + using AccType = typename PoolingTypes::AccType; CUDA_KERNEL_LOOP(index, nthreads) { - const int pw = index % pooled_width; - const int c = (index / pooled_width) % channels; + const bool nwc_layout = layout == mshadow::kNWC; + const int idx = nwc_layout ? (index / channels) : index; + const int pw = idx % pooled_width; + const int c = nwc_layout ? (index % channels) : (index / pooled_width) % channels; const int n = index / pooled_width / channels; int wstart = pw * stride_w - pad_w; int wend = min(wstart + kernel_w, width + pad_w); @@ -227,20 +242,22 @@ __global__ void pool_sum_1d_gpu_kernel(const int nthreads, const DType* in_data, if (get_avg && !count_include_pad) { pool_size = (wend - wstart); } - DType sum = 0; - const DType* out_slice = in_data + (n * channels + c) * width; + AccType sum = 0; + const DType* out_slice = nwc_layout ? in_data + n * channels * width + c + : in_data + (n * channels + c) * width; + const int multiplier = nwc_layout ? channels : 1; for (int w = wstart; w < wend; ++w) { - sum += a_pow_p::Map(out_slice[w]) / pool_size; + sum += a_pow_p::Map(out_slice[w * multiplier]) / pool_size; } - out_data[index] = a_root_p::Map(sum); + out_data[index] = a_root_p::Map(sum); } } /*! - * \brief avg/sum pooling gpu kernel for 2-D images. + * \brief avg/sum pooling gpu kernel for 2-D images, for both NCHW and NHWC layouts. * Do not call this kernel directly. Use the interface pool(). */ -template +template __global__ void pool_sum_2d_gpu_kernel(const int nthreads, const DType* in_data, const int channels, const int height, const int width, const int pooled_height, const int pooled_width, @@ -249,10 +266,14 @@ __global__ void pool_sum_2d_gpu_kernel(const int nthreads, const DType* in_data, const int pad_h, const int pad_w, DType* out_data, const bool get_avg = false, const bool count_include_pad = true) { + using AccType = typename PoolingTypes::AccType; CUDA_KERNEL_LOOP(index, nthreads) { - const int pw = index % pooled_width; - const int ph = (index / pooled_width) % pooled_height; - const int c = (index / pooled_width / pooled_height) % channels; + const bool nhwc_layout = layout == mshadow::kNHWC; + const int idx = nhwc_layout ? (index / channels) : index; + const int pw = idx % pooled_width; + const int ph = (idx / pooled_width) % pooled_height; + const int c = nhwc_layout ? (index % channels) + : (index / pooled_width / pooled_height) % channels; const int n = index / pooled_width / pooled_height / channels; int hstart = ph * stride_h - pad_h; int wstart = pw * stride_w - pad_w; @@ -266,22 +287,24 @@ __global__ void pool_sum_2d_gpu_kernel(const int nthreads, const DType* in_data, if (get_avg && !count_include_pad) { pool_size = (hend - hstart) * (wend - wstart); } - DType sum = 0; - const DType* out_slice = in_data + (n * channels + c) * height * width; + AccType sum = 0; + const DType* out_slice = nhwc_layout ? in_data + n * channels * height * width + c + : in_data + (n * channels + c) * height * width; + const int multiplier = nhwc_layout ? channels : 1; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - sum += a_pow_p::Map(out_slice[h * width + w]) / pool_size; + sum += a_pow_p::Map(out_slice[(h * width + w) * multiplier]) / pool_size; } } - out_data[index] = a_root_p::Map(sum); + out_data[index] = a_root_p::Map(sum); } } /*! - * \brief avg/sum pooling gpu kernel for 3-D images. + * \brief avg/sum pooling gpu kernel for 3-D images, for both NCDHW and NDHWC layouts. * Do not call this kernel directly. Use the interface pool(). */ -template +template __global__ void pool_sum_3d_gpu_kernel(const int nthreads, const DType* in_data, const int channels, const int depth, const int height, const int width, const int pooled_depth, const int pooled_height, @@ -291,11 +314,15 @@ __global__ void pool_sum_3d_gpu_kernel(const int nthreads, const DType* in_data, const int pad_d, const int pad_h, const int pad_w, DType* out_data, const bool get_avg = false, const bool count_include_pad = true) { + using AccType = typename PoolingTypes::AccType; CUDA_KERNEL_LOOP(index, nthreads) { - const int pw = index % pooled_width; - const int ph = (index / pooled_width) % pooled_height; - const int pd = (index / pooled_width / pooled_height) % pooled_depth; - const int c = (index / pooled_width / pooled_height / pooled_depth) % channels; + const bool ndhwc_layout = layout == mshadow::kNDHWC; + const int idx = ndhwc_layout ? (index / channels) : index; + const int pw = idx % pooled_width; + const int ph = (idx / pooled_width) % pooled_height; + const int pd = (idx / pooled_width / pooled_height) % pooled_depth; + const int c = ndhwc_layout ? (index % channels) + : (index / pooled_width / pooled_height / pooled_depth) % channels; const int n = index / pooled_width / pooled_height / pooled_depth / channels; int dstart = pd * stride_d - pad_d; int hstart = ph * stride_h - pad_h; @@ -313,51 +340,57 @@ __global__ void pool_sum_3d_gpu_kernel(const int nthreads, const DType* in_data, if (get_avg && !count_include_pad) { pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); } - DType sum = 0; - const DType* out_slice = in_data + (n * channels + c) * depth * height * width; + AccType sum = 0; + const DType* out_slice = ndhwc_layout ? in_data + n * channels * depth * height * width + c + : in_data + (n * channels + c) * depth * height * width; + const int multiplier = ndhwc_layout ? channels : 1; for (int d = dstart; d < dend; ++d) { for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - sum += a_pow_p::Map(out_slice[(d * height + h) * width + w]) / pool_size; + sum += a_pow_p::Map(out_slice[((d * height + h) * width + w) * + multiplier]) / pool_size; } } } out_data[index] = (pool_size == 0) ? - DType(nanf("")) : - a_root_p::Map(sum); + AccType(nanf("")) : + a_root_p::Map(sum); } } /*! - * \brief max unpooling gpu kernel for 1-D images. + * \brief max unpooling gpu kernel for 1-D images, for both NCW and NWC layouts. * Do not call this kernel directly. Use the interface unpool(). */ -template +template __global__ void unpool_max_1d_gpu_kernel(const int nthreads, const DType* out_grad, const DType* in_data, const DType* out_data, const int channels, const int width, const int pooled_width, const int kernel_w, const int stride_w, const int pad_w, DType* in_grad) { - // index is the output image's pixel index in NCHW + // index is the output image's pixel index // the order has to be consistent with pooling max // to avoid adding out_grad to the wrong in_grad // in the case where there are multiple max pixels // covered by a kernel window CUDA_KERNEL_LOOP(index, nthreads) { - const int pw = index % pooled_width; - const int c = (index / pooled_width) % channels; - const int n = index / pooled_width / channels; + const bool nwc_layout = layout == mshadow::kNWC; + const int idx = nwc_layout ? (index / channels) : index; + const int pw = idx % pooled_width; + const int c = nwc_layout ? index % channels : (index / pooled_width) % channels; + const int n = index / channels / pooled_width; int wstart = pw * stride_w - pad_w; const int wend = min(wstart + kernel_w, width); wstart = max(wstart, 0); // in data/grad offset batch and channel dims - int in_offset = (n * channels + c) * width; + const int in_offset = nwc_layout ? n * channels * width + c : (n * channels + c) * width; const DType* in_data_slice = in_data + in_offset; int max_idx = -1; DType max_val = out_data[index]; + const int multiplier = nwc_layout ? channels : 1; for (int w = wstart; w < wend; ++w) { - if (in_data_slice[w] == max_val) { + if (in_data_slice[w * multiplier] == max_val) { max_idx = w; break; } @@ -366,16 +399,16 @@ __global__ void unpool_max_1d_gpu_kernel(const int nthreads, const DType* out_gr // In the case where pad > 0 and kernel = 1, for example, // max_idx can be -1 reaching this step. if (max_idx >= 0) { - atomicAdd(&in_grad[in_offset+max_idx], out_grad[index]); + atomicAdd(&in_grad[in_offset + max_idx * multiplier], out_grad[index]); } } } /*! - * \brief max unpooling gpu kernel for 2-D images. + * \brief max unpooling gpu kernel for 2-D images, for both NCHW and NHWC layouts. * Do not call this kernel directly. Use the interface unpool(). */ -template +template __global__ void unpool_max_2d_gpu_kernel(const int nthreads, const DType* out_grad, const DType* in_data, const DType* out_data, const int channels, const int height, const int width, @@ -384,15 +417,18 @@ __global__ void unpool_max_2d_gpu_kernel(const int nthreads, const DType* out_gr const int stride_h, const int stride_w, const int pad_h, const int pad_w, DType* in_grad) { - // index is the output image's pixel index in NCHW + // index is the output image's pixel index // the order has to be consistent with pooling max // to avoid adding out_grad to the wrong in_grad // in the case where there are multiple max pixels // covered by a kernel window CUDA_KERNEL_LOOP(index, nthreads) { - const int pw = index % pooled_width; - const int ph = (index / pooled_width) % pooled_height; - const int c = (index / pooled_width / pooled_height) % channels; + const bool nhwc_layout = layout == mshadow::kNHWC; + const int idx = nhwc_layout ? (index / channels) : index; + const int pw = idx % pooled_width; + const int ph = (idx / pooled_width) % pooled_height; + const int c = nhwc_layout ? (index % channels) + : (index / pooled_width / pooled_height) % channels; const int n = index / pooled_width / pooled_height / channels; int hstart = ph * stride_h - pad_h; int wstart = pw * stride_w - pad_w; @@ -401,15 +437,17 @@ __global__ void unpool_max_2d_gpu_kernel(const int nthreads, const DType* out_gr hstart = max(hstart, 0); wstart = max(wstart, 0); // in data/grad offset batch and channel dims - int in_offset = (n * channels + c) * height * width; + int in_offset = nhwc_layout ? n * channels * height * width + c + : (n * channels + c) * height * width; const DType* in_data_slice = in_data + in_offset; int max_idx = -1; DType max_val = out_data[index]; + const int multiplier = nhwc_layout ? channels : 1; bool found = false; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { const int idx = h * width + w; - if (in_data_slice[idx] == max_val) { + if (in_data_slice[idx * multiplier] == max_val) { max_idx = idx; found = true; break; @@ -421,16 +459,16 @@ __global__ void unpool_max_2d_gpu_kernel(const int nthreads, const DType* out_gr // In the case where pad > 0 and kernel = 1, for example, // max_idx can be -1 reaching this step. if (max_idx >= 0) { - atomicAdd(&in_grad[in_offset+max_idx], out_grad[index]); + atomicAdd(&in_grad[in_offset + max_idx * multiplier], out_grad[index]); } } } /*! - * \brief max unpooling gpu kernel for 3-D images. + * \brief max unpooling gpu kernel for 3-D images, for both NCDHW and NDHWC layouts. * Do not call this kernel directly. Use the interface unpool(). */ -template +template __global__ void unpool_max_3d_gpu_kernel(const int nthreads, const DType* out_grad, const DType* in_data, const DType* out_data, const int channels, const int depth, const int height, @@ -441,16 +479,19 @@ __global__ void unpool_max_3d_gpu_kernel(const int nthreads, const DType* out_gr const int stride_h, const int stride_w, const int pad_d, const int pad_h, const int pad_w, DType* in_grad) { - // index is the output image's pixel index in NCDHW + // index is the output image's pixel index // the order has to be consistent with pooling max // to avoid adding out_grad to the wrong in_grad // in the case where there are multiple max pixels // covered by a kernel window CUDA_KERNEL_LOOP(index, nthreads) { - const int pw = index % pooled_width; - const int ph = (index / pooled_width) % pooled_height; - const int pd = (index / pooled_width / pooled_height) % pooled_depth; - const int c = (index / pooled_width / pooled_height / pooled_depth) % channels; + const bool ndhwc_layout = layout == mshadow::kNDHWC; + const int idx = ndhwc_layout ? (index / channels) : index; + const int pw = idx % pooled_width; + const int ph = (idx / pooled_width) % pooled_height; + const int pd = (idx / pooled_width / pooled_height) % pooled_depth; + const int c = ndhwc_layout ? (index % channels) + : (index / pooled_width / pooled_height / pooled_depth) % channels; const int n = index / pooled_width / pooled_height / pooled_depth / channels; int dstart = pd * stride_d - pad_d; int hstart = ph * stride_h - pad_h; @@ -462,16 +503,18 @@ __global__ void unpool_max_3d_gpu_kernel(const int nthreads, const DType* out_gr hstart = max(hstart, 0); wstart = max(wstart, 0); // in data/grad offset batch and channel dims - int in_offset = (n * channels + c) * depth * height * width; + int in_offset = ndhwc_layout ? n * channels * depth * height * width + c + : (n * channels + c) * depth * height * width; const DType* in_data_slice = in_data + in_offset; int max_idx = -1; DType max_val = out_data[index]; + const int multiplier = ndhwc_layout ? channels : 1; bool found = false; for (int d = dstart; d < dend; ++d) { for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { const int idx = (d * height + h) * width + w; - if (in_data_slice[idx] == max_val) { + if (in_data_slice[idx * multiplier] == max_val) { max_idx = idx; found = true; break; @@ -485,16 +528,16 @@ __global__ void unpool_max_3d_gpu_kernel(const int nthreads, const DType* out_gr // In the case where pad > 0 and kernel = 1, for example, // max_idx can be -1 reaching this step. if (max_idx >= 0) { - atomicAdd(&in_grad[in_offset+max_idx], out_grad[index]); + atomicAdd(&in_grad[in_offset + max_idx * multiplier], out_grad[index]); } } } /*! - * \brief avg/sum unpooling gpu kernel for 1-D images. + * \brief avg/sum unpooling gpu kernel for 1-D images, for both NCW and NWC layouts. * Do not call this kernel directly. Use the interface unpool(). */ -template +template __global__ void unpool_sum_1d_gpu_kernel(const int nthreads, const DType* out_grad, const DType* in_data, const DType* out_data, const int channels, const int width, @@ -502,20 +545,23 @@ __global__ void unpool_sum_1d_gpu_kernel(const int nthreads, const DType* out_gr const int stride_w, const int pad_w, DType* in_grad, const bool is_avg = false, const bool count_include_pad = true) { - // index is the input image index in NCW + // index is the input image index CUDA_KERNEL_LOOP(index, nthreads) { // find out the local index // find out the local offset - const int w = index % width + pad_w; - const int c = (index / width) % channels; + const bool nwc_layout = layout == mshadow::kNWC; + const int idx = nwc_layout ? (index / channels) : index; + const int w = idx % width + pad_w; + const int c = nwc_layout ? index % channels : (index / width) % channels; const int n = index / width / channels; const int pwstart = (w < kernel_w) ? 0 : (w - kernel_w) / stride_w + 1; const int pwend = min(w / stride_w + 1, pooled_width); DType gradient = 0; - const DType* out_grad_slice = - out_grad + (n * channels + c) * pooled_width; - const DType* out_data_slice = - out_data + (n * channels + c) * pooled_width; + const int slice_offset = nwc_layout ? n * channels * pooled_width + c + : (n * channels + c) * pooled_width; + const DType* out_grad_slice = out_grad + slice_offset; + const DType* out_data_slice = out_data + slice_offset; + const int multiplier = nwc_layout ? channels : 1; for (int pw = pwstart; pw < pwend; ++pw) { // figure out the pooling size int wstart = pw * stride_w - pad_w; @@ -527,7 +573,8 @@ __global__ void unpool_sum_1d_gpu_kernel(const int nthreads, const DType* out_gr pool_size = (wend - wstart); } gradient += - lp_grad::Map(out_grad_slice[pw], in_data[index], out_data_slice[pw]) / pool_size; + lp_grad::Map(out_grad_slice[pw * multiplier], in_data[index], + out_data_slice[pw * multiplier]) / pool_size; } // if req=kWriteTo, in_grad has already been assigned zero values in unpool() // use "+=" here instead of "=" to accommodate when req=kAddTo @@ -536,10 +583,10 @@ __global__ void unpool_sum_1d_gpu_kernel(const int nthreads, const DType* out_gr } /*! - * \brief avg/sum unpooling gpu kernel for 2-D images. + * \brief avg/sum unpooling gpu kernel for 2-D images, for both NCHW and NHWC layouts. * Do not call this kernel directly. Use the interface unpool(). */ -template +template __global__ void unpool_sum_2d_gpu_kernel(const int nthreads, const DType* out_grad, const DType* in_data, const DType* out_data, const int channels, const int height, const int width, @@ -549,23 +596,26 @@ __global__ void unpool_sum_2d_gpu_kernel(const int nthreads, const DType* out_gr const int pad_h, const int pad_w, DType* in_grad, const bool is_avg = false, const bool count_include_pad = true) { - // index is the input image index in NCHW + // index is the input image index CUDA_KERNEL_LOOP(index, nthreads) { // find out the local index // find out the local offset - const int w = index % width + pad_w; - const int h = (index / width) % height + pad_h; - const int c = (index / width / height) % channels; + const bool nhwc_layout = layout == mshadow::kNHWC; + const int idx = nhwc_layout ? (index / channels) : index; + const int w = idx % width + pad_w; + const int h = (idx / width) % height + pad_h; + const int c = nhwc_layout ? index % channels : (index / width / height) % channels; const int n = index / width / height / channels; const int phstart = (h < kernel_h) ? 0 : (h - kernel_h) / stride_h + 1; const int phend = min(h / stride_h + 1, pooled_height); const int pwstart = (w < kernel_w) ? 0 : (w - kernel_w) / stride_w + 1; const int pwend = min(w / stride_w + 1, pooled_width); DType gradient = 0; - const DType* out_grad_slice = - out_grad + (n * channels + c) * pooled_height * pooled_width; - const DType* out_data_slice = - out_data + (n * channels + c) * pooled_height * pooled_width; + const int slice_offset = nhwc_layout ? n * channels * pooled_height * pooled_width + c + : (n * channels + c) * pooled_height * pooled_width; + const DType* out_grad_slice = out_grad + slice_offset; + const DType* out_data_slice = out_data + slice_offset; + const int multiplier = nhwc_layout ? channels : 1; for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { // figure out the pooling size @@ -583,9 +633,9 @@ __global__ void unpool_sum_2d_gpu_kernel(const int nthreads, const DType* out_gr pool_size = (hend - hstart) * (wend - wstart); } gradient += - lp_grad::Map(out_grad_slice[out_index], + lp_grad::Map(out_grad_slice[out_index * multiplier], in_data[index], - out_data_slice[out_index]) / pool_size; + out_data_slice[out_index * multiplier]) / pool_size; } } // if req=kWriteTo, in_grad has already been assigned zero values in unpool() @@ -595,10 +645,10 @@ __global__ void unpool_sum_2d_gpu_kernel(const int nthreads, const DType* out_gr } /*! - * \brief avg/sum unpooling gpu kernel for 3-D images. + * \brief avg/sum unpooling gpu kernel for 3-D images, for both NCDHW and NDHWC layouts. * Do not call this kernel directly. Use the interface unpool(). */ -template +template __global__ void unpool_sum_3d_gpu_kernel(const int nthreads, const DType* out_grad, const DType* in_data, const DType* out_data, const int channels, const int depth, const int height, @@ -609,14 +659,16 @@ __global__ void unpool_sum_3d_gpu_kernel(const int nthreads, const DType* out_gr const int stride_w, const int pad_d, const int pad_h, const int pad_w, DType* in_grad, const bool is_avg = false, const bool count_include_pad = true) { - // index is the input image index in NCDHW + // index is the input image index CUDA_KERNEL_LOOP(index, nthreads) { // find out the local index // find out the local offset - const int w = index % width + pad_w; - const int h = (index / width) % height + pad_h; - const int d = (index / width / height) % depth + pad_d; - const int c = (index / width / height / depth) % channels; + const bool ndhwc_layout = layout == mshadow::kNDHWC; + const int idx = ndhwc_layout ? (index / channels) : index; + const int w = idx % width + pad_w; + const int h = (idx / width) % height + pad_h; + const int d = (idx / width / height) % depth + pad_d; + const int c = ndhwc_layout ? index % channels : (index / width / height / depth) % channels; const int n = index / width / height / depth / channels; const int pdstart = (d < kernel_d) ? 0 : (d - kernel_d) / stride_d + 1; const int pdend = min(d / stride_d + 1, pooled_depth); @@ -625,10 +677,12 @@ __global__ void unpool_sum_3d_gpu_kernel(const int nthreads, const DType* out_gr const int pwstart = (w < kernel_w) ? 0 : (w - kernel_w) / stride_w + 1; const int pwend = min(w / stride_w + 1, pooled_width); DType gradient = 0; - const DType* out_grad_slice = - out_grad + (n * channels + c) * pooled_depth * pooled_height * pooled_width; - const DType* out_data_slice = - out_data + (n * channels + c) * pooled_depth * pooled_height * pooled_width; + const int slice_offset = + ndhwc_layout ? n * channels * pooled_depth * pooled_height * pooled_width + c + : (n * channels + c) * pooled_depth * pooled_height * pooled_width; + const DType* out_grad_slice = out_grad + slice_offset; + const DType* out_data_slice = out_data + slice_offset; + const int multiplier = ndhwc_layout ? channels : 1; for (int pd = pdstart; pd < pdend; ++pd) { for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { @@ -650,9 +704,9 @@ __global__ void unpool_sum_3d_gpu_kernel(const int nthreads, const DType* out_gr wend = min(wend, width); pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); } - gradient += lp_grad::Map(out_grad_slice[out_index], + gradient += lp_grad::Map(out_grad_slice[out_index * multiplier], in_data[index], - out_data_slice[out_index]) / pool_size; + out_data_slice[out_index * multiplier]) / pool_size; } } } @@ -674,9 +728,9 @@ __global__ void unpool_sum_3d_gpu_kernel(const int nthreads, const DType* out_gr * \param pool_type supported pooling type: max, avg, sum * \param req_type operator request type, only support kWriteTo for now * \param out_data pointer of the output tensor data in the format of NCW, NCHW, or NCDHW - * \param p_value value of p for Lp pooling + * \param count_include_pad for avg pooling, should 0 pad values be averaged in the window */ -template +template inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& ishape, const TShape& oshape, const TShape& kernel, const TShape& pad, const TShape& stride, const int pool_type, OpReqType req_type, @@ -686,14 +740,14 @@ inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& is if (kernel.ndim() == 1) { if (pool_enum::kMaxPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - pool_max_1d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), in_data, ishape[1], ishape[2], oshape[2], kernel[0], stride[0], pad[0], out_data); MSHADOW_CUDA_POST_KERNEL_CHECK(pool_max_1d_gpu_kernel); } else if (pool_enum::kAvgPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - pool_sum_1d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), in_data, ishape[1], ishape[2], oshape[2], kernel[0], stride[0], pad[0], out_data, @@ -701,14 +755,14 @@ inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& is MSHADOW_CUDA_POST_KERNEL_CHECK(pool_sum_1d_gpu_kernel); } else if (pool_enum::kSumPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - pool_sum_1d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), in_data, ishape[1], ishape[2], oshape[2], kernel[0], stride[0], pad[0], out_data); MSHADOW_CUDA_POST_KERNEL_CHECK(pool_sum_1d_gpu_kernel); } else if (pool_enum::kLpPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - pool_sum_1d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), in_data, ishape[1], ishape[2], oshape[2], kernel[0], stride[0], pad[0], out_data); @@ -719,7 +773,7 @@ inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& is } else if (kernel.ndim() == 2) { if (pool_enum::kMaxPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - pool_max_2d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), in_data, ishape[1], ishape[2], ishape[3], oshape[2], oshape[3], kernel[0], kernel[1], @@ -727,7 +781,7 @@ inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& is MSHADOW_CUDA_POST_KERNEL_CHECK(pool_max_2d_gpu_kernel); } else if (pool_enum::kAvgPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - pool_sum_2d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), in_data, ishape[1], ishape[2], ishape[3], oshape[2], oshape[3], kernel[0], kernel[1], @@ -736,7 +790,7 @@ inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& is MSHADOW_CUDA_POST_KERNEL_CHECK(pool_sum_2d_gpu_kernel); } else if (pool_enum::kSumPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - pool_sum_2d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), in_data, ishape[1], ishape[2], ishape[3], oshape[2], oshape[3], kernel[0], kernel[1], @@ -744,7 +798,7 @@ inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& is MSHADOW_CUDA_POST_KERNEL_CHECK(pool_sum_2d_gpu_kernel); } else if (pool_enum::kLpPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - pool_sum_2d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), in_data, ishape[1], ishape[2], ishape[3], oshape[2], oshape[3], kernel[0], kernel[1], @@ -756,7 +810,7 @@ inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& is } else if (kernel.ndim() == 3) { if (pool_enum::kMaxPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - pool_max_3d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), in_data, ishape[1], ishape[2], ishape[3], ishape[4], oshape[2], oshape[3], oshape[4], @@ -765,7 +819,7 @@ inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& is MSHADOW_CUDA_POST_KERNEL_CHECK(pool_max_3d_gpu_kernel); } else if (pool_enum::kAvgPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - pool_sum_3d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), in_data, ishape[1], ishape[2], ishape[3], ishape[4], oshape[2], oshape[3], oshape[4], kernel[0], @@ -774,7 +828,7 @@ inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& is MSHADOW_CUDA_POST_KERNEL_CHECK(pool_sum_3d_gpu_kernel); } else if (pool_enum::kSumPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - pool_sum_3d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), in_data, ishape[1], ishape[2], ishape[3], ishape[4], oshape[2], oshape[3], oshape[4], kernel[0], @@ -783,7 +837,7 @@ inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& is MSHADOW_CUDA_POST_KERNEL_CHECK(pool_sum_3d_gpu_kernel); } else if (pool_enum::kLpPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - pool_sum_3d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), in_data, ishape[1], ishape[2], ishape[3], ishape[4], oshape[2], oshape[3], oshape[4], kernel[0], @@ -796,6 +850,70 @@ inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& is } } +/*! + * \brief This function serves as an interface for 1/2/3-D pooling operations. + * \param s context stream defining the device in use is gpu + * \param in_data pointer of the input tensor data + * \param ishape input tensor shape + * \param oshape output tensor shape + * \param kernel kernel shape + * \param pad pad shape + * \param stride stride shape + * \param pool_type supported pooling type: max, avg, sum + * \param req_type operator request type, only support kWriteTo for now + * \param out_data pointer of the output tensor data + * \param count_include_pad for avg pooling, should 0 pad values be averaged in the window + * \param layout I/O tensor layout, e.g. NCHW vs. NHWC + */ +template +inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& ishape, + const TShape& oshape, const TShape& kernel, const TShape& pad, + const TShape& stride, const int pool_type, OpReqType req_type, + DType* out_data, const bool count_include_pad, int layout) { + if (kernel.ndim() == 1) { + if (layout == mshadow::kNWC) { + // standardize shapes to NCW to aid templated kernel invocation + TShape ishape_ncw = ConvertLayout(ishape.get<3>(), mshadow::kNWC, mshadow::kNCW); + TShape oshape_ncw = ConvertLayout(oshape.get<3>(), mshadow::kNWC, mshadow::kNCW); + pool(s, in_data, ishape_ncw, oshape_ncw, kernel, + pad, stride, pool_type, req_type, out_data, count_include_pad); + } else if (layout == mshadow::kNCW) { + pool(s, in_data, ishape, oshape, kernel, + pad, stride, pool_type, req_type, out_data, count_include_pad); + } else { + LOG(FATAL) << "Unsupported layout, expecting kNCW or kNWC, saw: " << layout; + } + } else if (kernel.ndim() == 2) { + if (layout == mshadow::kNHWC) { + // standardize shapes to NCHW to aid templated kernel invocation + TShape ishape_nchw = ConvertLayout(ishape.get<4>(), mshadow::kNHWC, mshadow::kNCHW); + TShape oshape_nchw = ConvertLayout(oshape.get<4>(), mshadow::kNHWC, mshadow::kNCHW); + pool(s, in_data, ishape_nchw, oshape_nchw, kernel, + pad, stride, pool_type, req_type, out_data, count_include_pad); + } else if (layout == mshadow::kNCHW) { + pool(s, in_data, ishape, oshape, kernel, + pad, stride, pool_type, req_type, out_data, count_include_pad); + } else { + LOG(FATAL) << "Unsupported layout, expecting kNCHW or kNHWC, saw: " << layout; + } + } else if (kernel.ndim() == 3) { + if (layout == mshadow::kNDHWC) { + // standardize shapes to NCDHW to aid templated kernel invocation + TShape ishape_ncdhw = ConvertLayout(ishape.get<5>(), mshadow::kNDHWC, mshadow::kNCDHW); + TShape oshape_ncdhw = ConvertLayout(oshape.get<5>(), mshadow::kNDHWC, mshadow::kNCDHW); + pool(s, in_data, ishape_ncdhw, oshape_ncdhw, kernel, + pad, stride, pool_type, req_type, out_data, count_include_pad); + } else if (layout == mshadow::kNCDHW) { + pool(s, in_data, ishape, oshape, kernel, + pad, stride, pool_type, req_type, out_data, count_include_pad); + } else { + LOG(FATAL) << "Unsupported layout, expecting kNCDHW or kNDHWC, saw: " << layout; + } + } else { + LOG(FATAL) << "Unsupported " << kernel.ndim() << "-D pooling"; + } +} + /*! * \brief This function serves as an interface for 1/2/3-D unpooling operations. * \param s context stream defining the device in use is gpu @@ -810,9 +928,9 @@ inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& is * \param pool_type supported pooling type: max, avg, sum * \param req_type operator request type: kNullOp, kNullWriteInplace, kNullWriteTo, kNullAddTo * \param in_grad pointer of the gradient of the operator's input tensor - * \param p_value value of p for Lp pooling + * \param count_include_pad for avg pooling, should 0 pad values be averaged in the window */ -template +template inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* in_data, const DType* out_data, const TShape& ishape, const TShape& oshape, const TShape& kernel, const TShape& pad, const TShape& stride, @@ -826,7 +944,7 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* if (kernel.ndim() == 1) { if (pool_enum::kMaxPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - unpool_max_1d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), out_grad, in_data, out_data, ishape[1], ishape[2], oshape[2], kernel[0], stride[0], pad[0], @@ -834,7 +952,7 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* MSHADOW_CUDA_POST_KERNEL_CHECK(unpool_max_1d_gpu_kernel); } else if (pool_enum::kAvgPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - unpool_sum_1d_gpu_kernel<<<<::GetStream(s)>>>( ishape.Size(), out_grad, in_data, out_data, ishape[1], ishape[2], oshape[2], kernel[0], @@ -842,7 +960,7 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* MSHADOW_CUDA_POST_KERNEL_CHECK(unpool_sum_1d_gpu_kernel); } else if (pool_enum::kSumPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - unpool_sum_1d_gpu_kernel<<<<::GetStream(s)>>>( ishape.Size(), out_grad, in_data, out_data, ishape[1], ishape[2], oshape[2], kernel[0], @@ -850,7 +968,7 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* MSHADOW_CUDA_POST_KERNEL_CHECK(unpool_sum_1d_gpu_kernel); } else if (pool_enum::kLpPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - unpool_sum_1d_gpu_kernel<<<<::GetStream(s)>>>( ishape.Size(), out_grad, in_data, out_data, ishape[1], ishape[2], oshape[2], kernel[0], @@ -862,7 +980,7 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* } else if (kernel.ndim() == 2) { if (pool_enum::kMaxPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - unpool_max_2d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), out_grad, in_data, out_data, ishape[1], ishape[2], ishape[3], @@ -871,7 +989,7 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* MSHADOW_CUDA_POST_KERNEL_CHECK(unpool_max_2d_gpu_kernel); } else if (pool_enum::kAvgPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - unpool_sum_2d_gpu_kernel<<<<::GetStream(s)>>>( ishape.Size(), out_grad, in_data, out_data, ishape[1], ishape[2], ishape[3], @@ -881,7 +999,7 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* MSHADOW_CUDA_POST_KERNEL_CHECK(unpool_sum_2d_gpu_kernel); } else if (pool_enum::kSumPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - unpool_sum_2d_gpu_kernel<<<<::GetStream(s)>>>( ishape.Size(), out_grad, in_data, out_data, ishape[1], ishape[2], ishape[3], @@ -890,7 +1008,7 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* MSHADOW_CUDA_POST_KERNEL_CHECK(unpool_sum_2d_gpu_kernel); } else if (pool_enum::kLpPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - unpool_sum_2d_gpu_kernel<<<<::GetStream(s)>>>( ishape.Size(), out_grad, in_data, out_data, ishape[1], ishape[2], ishape[3], @@ -903,7 +1021,7 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* } else if (kernel.ndim() == 3) { if (pool_enum::kMaxPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - unpool_max_3d_gpu_kernel<<<<::GetStream(s)>>>( oshape.Size(), out_grad, in_data, out_data, ishape[1], ishape[2], ishape[3], ishape[4], @@ -913,7 +1031,7 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* MSHADOW_CUDA_POST_KERNEL_CHECK(unpool_max_3d_gpu_kernel); } else if (pool_enum::kAvgPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - unpool_sum_3d_gpu_kernel<<<<::GetStream(s)>>>( ishape.Size(), out_grad, in_data, out_data, ishape[1], ishape[2], ishape[3], ishape[4], @@ -923,7 +1041,7 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* MSHADOW_CUDA_POST_KERNEL_CHECK(unpool_sum_3d_gpu_kernel); } else if (pool_enum::kSumPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - unpool_sum_3d_gpu_kernel<<<<::GetStream(s)>>>( ishape.Size(), out_grad, in_data, out_data, ishape[1], ishape[2], ishape[3], ishape[4], @@ -933,7 +1051,7 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* MSHADOW_CUDA_POST_KERNEL_CHECK(unpool_sum_3d_gpu_kernel); } else if (pool_enum::kLpPooling == pool_type) { // NOLINT_NEXT_LINE(whitespace/operators) - unpool_sum_3d_gpu_kernel<<<<::GetStream(s)>>>( ishape.Size(), out_grad, in_data, out_data, ishape[1], ishape[2], ishape[3], ishape[4], @@ -949,6 +1067,73 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* } } +/*! + * \brief This function serves as an interface for 1/2/3-D unpooling operations. + * \param s context stream defining the device in use is gpu + * \param out_grad pointer of the gradient of operator's output tensor + * \param in_data pointer of the input tensor in the format of NCW, NCHW, or NCDHW + * \param out_data pointer of the output tensor in the format of NCW, NCHW, or NCDHW + * \param ishape input tensor shape + * \param oshape output tensor shape + * \param kernel kernel shape + * \param pad pad shape + * \param stride stride shape + * \param pool_type supported pooling type: max, avg, sum + * \param req_type operator request type: kNullOp, kNullWriteInplace, kNullWriteTo, kNullAddTo + * \param in_grad pointer of the gradient of the operator's input tensor + * \param count_include_pad for avg pooling, should 0 pad values be averaged in the window + * \param layout I/O tensor layout, e.g. NCHW vs. NHWC + */ +template +inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* in_data, + const DType* out_data, const TShape& ishape, const TShape& oshape, + const TShape& kernel, const TShape& pad, const TShape& stride, + const int pool_type, OpReqType req_type, DType* in_grad, + const bool count_include_pad, int layout) { + if (kernel.ndim() == 1) { + if (layout == mshadow::kNWC) { + // standardize shapes to NCW to aid templated kernel invocation + TShape ishape_ncw = ConvertLayout(ishape.get<3>(), mshadow::kNWC, mshadow::kNCW); + TShape oshape_ncw = ConvertLayout(oshape.get<3>(), mshadow::kNWC, mshadow::kNCW); + unpool(s, out_grad, in_data, out_data, ishape_ncw, oshape_ncw, + kernel, pad, stride, pool_type, req_type, in_grad, count_include_pad); + } else if (layout == mshadow::kNCW) { + unpool(s, out_grad, in_data, out_data, ishape, oshape, kernel, + pad, stride, pool_type, req_type, in_grad, count_include_pad); + } else { + LOG(FATAL) << "Unsupported layout, expecting kNCW or kNWC, saw: " << layout; + } + } else if (kernel.ndim() == 2) { + if (layout == mshadow::kNHWC) { + // standardize shapes to NCHW to aid templated kernel invocation + TShape ishape_nchw = ConvertLayout(ishape.get<4>(), mshadow::kNHWC, mshadow::kNCHW); + TShape oshape_nchw = ConvertLayout(oshape.get<4>(), mshadow::kNHWC, mshadow::kNCHW); + unpool(s, out_grad, in_data, out_data, ishape_nchw, oshape_nchw, + kernel, pad, stride, pool_type, req_type, in_grad, count_include_pad); + } else if (layout == mshadow::kNCHW) { + unpool(s, out_grad, in_data, out_data, ishape, oshape, kernel, + pad, stride, pool_type, req_type, in_grad, count_include_pad); + } else { + LOG(FATAL) << "Unsupported layout, expecting kNCHW or kNHWC, saw: " << layout; + } + } else if (kernel.ndim() == 3) { + if (layout == mshadow::kNDHWC) { + // standardize shapes to NCDHW to aid templated kernel invocation + TShape ishape_ncdhw = ConvertLayout(ishape.get<5>(), mshadow::kNDHWC, mshadow::kNCDHW); + TShape oshape_ncdhw = ConvertLayout(oshape.get<5>(), mshadow::kNDHWC, mshadow::kNCDHW); + unpool(s, out_grad, in_data, out_data, ishape_ncdhw, oshape_ncdhw, + kernel, pad, stride, pool_type, req_type, in_grad, count_include_pad); + } else if (layout == mshadow::kNCDHW) { + unpool(s, out_grad, in_data, out_data, ishape, oshape, kernel, + pad, stride, pool_type, req_type, in_grad, count_include_pad); + } else { + LOG(FATAL) << "Unsupported layout, expecting kNCDHW or kNDHWC, saw: " << layout; + } + } else { + LOG(FATAL) << "Unsupported " << kernel.ndim() << "-D unpooling"; + } +} + } // namespace op } // namespace mxnet diff --git a/src/operator/nn/pool.h b/src/operator/nn/pool.h index 33005c8e5f0f..3c8c19a02607 100644 --- a/src/operator/nn/pool.h +++ b/src/operator/nn/pool.h @@ -61,6 +61,7 @@ #include #include +#include #include #include "./pool_utils.h" #include "../mxnet_op.h" @@ -77,13 +78,13 @@ enum PoolingOpPadConventionType {kValid, kFull, kSame}; } // namespace pool_enum /*! - * \brief max pooling cpu function for 1-D images. + * \brief max pooling cpu function for 1-D images in 'ncw' layout. * Do not call this kernel directly. Use the interface pool(). */ template -inline void pool_max_1d_cpu(const DType* in_data, const TShape& ishape, const TShape& oshape, - const TShape& kernel, const TShape& pad, const TShape& stride, - DType* out_data) { +inline void pool_max_1d_ncw_cpu(const DType *in_data, const TShape &ishape, const TShape &oshape, + const TShape &kernel, const TShape &pad, const TShape &stride, + DType *out_data) { using mshadow::red::limits::MinValue; const int width = ishape[2]; const int pooled_width = oshape[2]; @@ -113,14 +114,53 @@ inline void pool_max_1d_cpu(const DType* in_data, const TShape& ishape, const TS } /*! - * \brief max pooling cpu function for 2-D images. + * \brief max pooling cpu function for 1-D images in 'nwc' layout. * Do not call this kernel directly. Use the interface pool(). */ template -inline void pool_max_2d_cpu(const DType* in_data, const TShape& ishape, const TShape& oshape, +inline void pool_max_1d_nwc_cpu(const DType* in_data, const TShape& ishape, const TShape& oshape, const TShape& kernel, const TShape& pad, const TShape& stride, DType* out_data) { using mshadow::red::limits::MinValue; + const int width = ishape[1]; + const int pooled_width = oshape[1]; + const int kernel_w = kernel[0]; + const int pad_w = pad[0]; + const int stride_w = stride[0]; + const int features = oshape[2]; + const index_t in_data_offset = ishape[1] * features; + const index_t out_data_offset = oshape[1] * features; + std::vector max_vals(features); + for (index_t n = 0; n < oshape[0]; ++n) { + for (int pw = 0; pw < pooled_width; ++pw) { + int wstart = pw * stride_w - pad_w; + int wend = std::min(wstart + kernel_w, width); + wstart = std::max(wstart, 0); + std::fill(max_vals.begin(), max_vals.end(), MinValue()); + for (int w = wstart; w < wend; ++w) { + for (index_t c = 0; c < features; ++c) { + if (in_data[w * features + c] > max_vals[c]) { + max_vals[c] = in_data[w * features + c]; + } + } + } + for (index_t c = 0; c < features; ++c) + out_data[pw * features + c] = max_vals[c]; + } + in_data += in_data_offset; + out_data += out_data_offset; + } +} + +/*! + * \brief max pooling cpu function for 2-D images in 'nchw' layout. + * Do not call this kernel directly. Use the interface pool(). + */ +template +inline void pool_max_2d_nchw_cpu(const DType *in_data, const TShape &ishape, const TShape &oshape, + const TShape &kernel, const TShape &pad, const TShape &stride, + DType *out_data) { + using mshadow::red::limits::MinValue; const int height = ishape[2], width = ishape[3]; const int pooled_height = oshape[2], pooled_width = oshape[3]; const int kernel_h = kernel[0], kernel_w = kernel[1]; @@ -158,14 +198,62 @@ inline void pool_max_2d_cpu(const DType* in_data, const TShape& ishape, const TS } /*! - * \brief max pooling cpu function for 3-D images. + * \brief max pooling cpu function for 2-D images in 'nhwc' layout. * Do not call this kernel directly. Use the interface pool(). */ template -inline void pool_max_3d_cpu(const DType* in_data, const TShape& ishape, const TShape& oshape, +inline void pool_max_2d_nhwc_cpu(const DType* in_data, const TShape& ishape, const TShape& oshape, const TShape& kernel, const TShape& pad, const TShape& stride, DType* out_data) { using mshadow::red::limits::MinValue; + const int height = ishape[1], width = ishape[2]; + const int pooled_height = oshape[1], pooled_width = oshape[2]; + const int kernel_h = kernel[0], kernel_w = kernel[1]; + const int pad_h = pad[0], pad_w = pad[1]; + const int stride_h = stride[0], stride_w = stride[1]; + const int features = oshape[3]; + const index_t in_data_offset = ishape[1] * ishape[2] * features; + const index_t out_data_offset = oshape[1] * oshape[2] * features; + std::vector max_vals(features); + for (index_t n = 0; n < oshape[0]; ++n) { + for (int ph = 0; ph < pooled_height; ++ph) { + for (int pw = 0; pw < pooled_width; ++pw) { + int hstart = ph * stride_h - pad_h; + int wstart = pw * stride_w - pad_w; + int hend = std::min(hstart + kernel_h, height); + int wend = std::min(wstart + kernel_w, width); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + const int pool_index = ph * pooled_width + pw; + std::fill(max_vals.begin(), max_vals.end(), MinValue()); + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + const int in_index = h * width + w; + for (index_t c = 0; c < features; ++c) { + if (in_data[in_index * features + c] > max_vals[c]) { + max_vals[c] = in_data[in_index * features + c]; + } + } + } + } + for (index_t c = 0; c < features; ++c) + out_data[pool_index * features + c] = max_vals[c]; + } + } + in_data += in_data_offset; + out_data += out_data_offset; + } +} + +/*! + * \brief max pooling cpu function for 3-D images in 'ncdhw' layout. + * Do not call this kernel directly. Use the interface pool(). + */ +template +inline void pool_max_3d_ncdhw_cpu(const DType *in_data, const TShape &ishape, const TShape &oshape, + const TShape &kernel, const TShape &pad, const TShape &stride, + DType *out_data) { + using mshadow::red::limits::MinValue; const int depth = ishape[2], height = ishape[3], width = ishape[4]; const int pooled_depth = oshape[2], pooled_height = oshape[3], pooled_width = oshape[4]; const int kernel_d = kernel[0], kernel_h = kernel[1], kernel_w = kernel[2]; @@ -210,14 +298,70 @@ inline void pool_max_3d_cpu(const DType* in_data, const TShape& ishape, const TS } /*! - * \brief avg/sum pooling cpu function for 1-D images. + * \brief max pooling cpu function for 3-D images in 'ndhwc' layout. * Do not call this kernel directly. Use the interface pool(). */ -template -inline void pool_sum_1d_cpu(const DType* in_data, const TShape& ishape, const TShape& oshape, +template +inline void pool_max_3d_ndhwc_cpu(const DType* in_data, const TShape& ishape, const TShape& oshape, const TShape& kernel, const TShape& pad, const TShape& stride, - DType* out_data, - const bool get_avg = false, const bool count_include_pad = true) { + DType* out_data) { + using mshadow::red::limits::MinValue; + const int depth = ishape[1], height = ishape[2], width = ishape[3]; + const int pooled_depth = oshape[1], pooled_height = oshape[2], pooled_width = oshape[3]; + const int kernel_d = kernel[0], kernel_h = kernel[1], kernel_w = kernel[2]; + const int pad_d = pad[0], pad_h = pad[1], pad_w = pad[2]; + const int stride_d = stride[0], stride_h = stride[1], stride_w = stride[2]; + const int features = oshape[4]; + const index_t in_data_offset = ishape[1] * ishape[2] * ishape[3] * features; + const index_t out_data_offset = oshape[1] * oshape[2] * oshape[3] * features; + std::vector max_vals(features); + for (index_t n = 0; n < oshape[0]; ++n) { + for (int pd = 0; pd < pooled_depth; ++pd) { + for (int ph = 0; ph < pooled_height; ++ph) { + for (int pw = 0; pw < pooled_width; ++pw) { + int dstart = pd * stride_d - pad_d; + int hstart = ph * stride_h - pad_h; + int wstart = pw * stride_w - pad_w; + int dend = std::min(dstart + kernel_d, depth); + int hend = std::min(hstart + kernel_h, height); + int wend = std::min(wstart + kernel_w, width); + dstart = std::max(dstart, 0); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + const int pool_index = (pd * pooled_height + ph) * pooled_width + pw; + std::fill(max_vals.begin(), max_vals.end(), MinValue()); + for (int d = dstart; d < dend; ++d) { + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + const int in_index = (d * height + h) * width + w; + for (index_t c = 0; c < features; ++c) { + if (in_data[in_index * features + c] > max_vals[c]) { + max_vals[c] = in_data[in_index * features + c]; + } + } + } + } + } + for (index_t c = 0; c < features; ++c) + out_data[pool_index * features + c] = max_vals[c]; + } + } + } + in_data += in_data_offset; + out_data += out_data_offset; + } +} + +/*! + * \brief avg/sum pooling cpu function for 1-D images in 'ncw' layout. + * Do not call this kernel directly. Use the interface pool(). + */ +template +inline void pool_sum_1d_ncw_cpu(const DType *in_data, const TShape &ishape, const TShape &oshape, + const TShape &kernel, const TShape &pad, const TShape &stride, + DType *out_data, + const bool get_avg = false, const bool count_include_pad = true) { + using AccType = typename PoolingTypes::AccType; const int width = ishape[2]; const int pooled_width = oshape[2]; const int kernel_w = kernel[0]; @@ -236,11 +380,11 @@ inline void pool_sum_1d_cpu(const DType* in_data, const TShape& ishape, const TS if (get_avg && !count_include_pad) { pool_size = (wend - wstart); } - DType sum = 0; + AccType sum = 0; for (int w = wstart; w < wend; ++w) { - sum += a_pow_p::Map(in_data[w]) / pool_size; + sum += a_pow_p::Map(in_data[w]) / pool_size; } - out_data[pw] = a_root_p::Map(sum); + out_data[pw] = a_root_p::Map(sum); } in_data += in_data_offset; out_data += out_data_offset; @@ -249,14 +393,58 @@ inline void pool_sum_1d_cpu(const DType* in_data, const TShape& ishape, const TS } /*! - * \brief avg/sum pooling cpu function for 2-D images. + * \brief avg/sum pooling cpu function for 1-D images in 'nwc' layout. * Do not call this kernel directly. Use the interface pool(). */ template -inline void pool_sum_2d_cpu(const DType* in_data, const TShape& ishape, const TShape& oshape, +inline void pool_sum_1d_nwc_cpu(const DType* in_data, const TShape& ishape, const TShape& oshape, const TShape& kernel, const TShape& pad, const TShape& stride, DType* out_data, const bool get_avg = false, const bool count_include_pad = true) { + using AccType = typename PoolingTypes::AccType; + const int width = ishape[1]; + const int pooled_width = oshape[1]; + const int kernel_w = kernel[0]; + const int pad_w = pad[0]; + const int stride_w = stride[0]; + const int features = oshape[2]; + const index_t in_data_offset = ishape[1] * features; + const index_t out_data_offset = oshape[1] * features; + std::vector sums(features); + for (index_t n = 0; n < oshape[0]; ++n) { + for (int pw = 0; pw < pooled_width; ++pw) { + int wstart = pw * stride_w - pad_w; + int wend = std::min(wstart + kernel_w, width + pad_w); + int pool_size = (get_avg ? (wend - wstart) : 1); + wstart = std::max(wstart, 0); + wend = std::min(wend, width); + if (get_avg && !count_include_pad) { + pool_size = (wend - wstart); + } + std::fill(sums.begin(), sums.end(), 0); + for (int w = wstart; w < wend; ++w) { + for (index_t c = 0; c < features; ++c) { + sums[c] += a_pow_p::Map(in_data[w * features + c]) / pool_size; + } + } + for (index_t c = 0; c < features; ++c) + out_data[pw * features + c] = a_root_p::Map(sums[c]); + } + in_data += in_data_offset; + out_data += out_data_offset; + } +} + +/*! + * \brief avg/sum pooling cpu function for 2-D images in 'nchw' layout. + * Do not call this kernel directly. Use the interface pool(). + */ +template +inline void pool_sum_2d_nchw_cpu(const DType *in_data, const TShape &ishape, const TShape &oshape, + const TShape &kernel, const TShape &pad, const TShape &stride, + DType *out_data, + const bool get_avg = false, const bool count_include_pad = true) { + using AccType = typename PoolingTypes::AccType; const int height = ishape[2], width = ishape[3]; const int pooled_height = oshape[2], pooled_width = oshape[3]; const int kernel_h = kernel[0], kernel_w = kernel[1]; @@ -280,13 +468,13 @@ inline void pool_sum_2d_cpu(const DType* in_data, const TShape& ishape, const TS if (get_avg && !count_include_pad) { pool_size = (hend - hstart) * (wend - wstart); } - DType sum = 0; + AccType sum = 0; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - sum += a_pow_p::Map(in_data[h*width+w]) / pool_size; + sum += a_pow_p::Map(in_data[h*width+w]) / pool_size; } } - out_data[ph*pooled_width+pw] = a_root_p::Map(sum); + out_data[ph*pooled_width+pw] = a_root_p::Map(sum); } } in_data += in_data_offset; @@ -296,14 +484,68 @@ inline void pool_sum_2d_cpu(const DType* in_data, const TShape& ishape, const TS } /*! - * \brief avg/sum pooling cpu function for 3-D images. + * \brief avg/sum pooling cpu function for 2-D images in 'nhwc' layout. * Do not call this kernel directly. Use the interface pool(). */ template -inline void pool_sum_3d_cpu(const DType* in_data, const TShape& ishape, const TShape& oshape, +inline void pool_sum_2d_nhwc_cpu(const DType* in_data, const TShape& ishape, const TShape& oshape, const TShape& kernel, const TShape& pad, const TShape& stride, DType* out_data, const bool get_avg = false, const bool count_include_pad = true) { + using AccType = typename PoolingTypes::AccType; + const int height = ishape[1], width = ishape[2]; + const int pooled_height = oshape[1], pooled_width = oshape[2]; + const int kernel_h = kernel[0], kernel_w = kernel[1]; + const int pad_h = pad[0], pad_w = pad[1]; + const int stride_h = stride[0], stride_w = stride[1]; + const int features = oshape[3]; + const index_t in_data_offset = ishape[1] * ishape[2] * features; + const index_t out_data_offset = oshape[1] * oshape[2] * features; + std::vector sums(features); + for (index_t n = 0; n < oshape[0]; ++n) { + for (int ph = 0; ph < pooled_height; ++ph) { + for (int pw = 0; pw < pooled_width; ++pw) { + int hstart = ph * stride_h - pad_h; + int wstart = pw * stride_w - pad_w; + int hend = std::min(hstart + kernel_h, height + pad_h); + int wend = std::min(wstart + kernel_w, width + pad_w); + int pool_size = (get_avg ? (hend - hstart) * (wend - wstart) : 1); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + hend = std::min(hend, height); + wend = std::min(wend, width); + if (get_avg && !count_include_pad) { + pool_size = (hend - hstart) * (wend - wstart); + } + const int pool_index = ph * pooled_width + pw; + std::fill(sums.begin(), sums.end(), 0); + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + const int in_index = h * width + w; + for (index_t c = 0; c < features; ++c) { + sums[c] += a_pow_p::Map(in_data[in_index * features + c]) / pool_size; + } + } + } + for (index_t c = 0; c < features; ++c) + out_data[pool_index * features + c] = a_root_p::Map(sums[c]); + } + } + in_data += in_data_offset; + out_data += out_data_offset; + } +} + +/*! + * \brief avg/sum pooling cpu function for 3-D images in 'ncdhw' layout. + * Do not call this kernel directly. Use the interface pool(). + */ +template +inline void pool_sum_3d_ncdhw_cpu(const DType *in_data, const TShape &ishape, const TShape &oshape, + const TShape &kernel, const TShape &pad, const TShape &stride, + DType *out_data, + const bool get_avg = false, const bool count_include_pad = true) { + using AccType = typename PoolingTypes::AccType; const int depth = ishape[2], height = ishape[3], width = ishape[4]; const int pooled_depth = oshape[2], pooled_height = oshape[3], pooled_width = oshape[4]; const int kernel_d = kernel[0], kernel_h = kernel[1], kernel_w = kernel[2]; @@ -332,17 +574,17 @@ inline void pool_sum_3d_cpu(const DType* in_data, const TShape& ishape, const TS if (get_avg && !count_include_pad) { pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); } - DType sum = 0; + AccType sum = 0; for (int d = dstart; d < dend; ++d) { for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - sum += a_pow_p::Map(in_data[(d*height+h)*width+w]) / pool_size; + sum += a_pow_p::Map(in_data[(d*height+h)*width+w]) / pool_size; } } } out_data[(pd*pooled_height+ph)*pooled_width+pw] = (pool_size == 0) ? - DType(nanf("")) : - a_root_p::Map(sum); + AccType(nanf("")) : + a_root_p::Map(sum); } } } @@ -353,15 +595,78 @@ inline void pool_sum_3d_cpu(const DType* in_data, const TShape& ishape, const TS } /*! - * \brief max unpooling cpu function for 1-D images. + * \brief avg/sum pooling cpu function for 3-D images in 'ndhwc' layout. + * Do not call this kernel directly. Use the interface pool(). + */ +template +inline void pool_sum_3d_ndhwc_cpu(const DType* in_data, const TShape& ishape, const TShape& oshape, + const TShape& kernel, const TShape& pad, const TShape& stride, + DType* out_data, + const bool get_avg = false, const bool count_include_pad = true) { + using AccType = typename PoolingTypes::AccType; + const int depth = ishape[1], height = ishape[2], width = ishape[3]; + const int pooled_depth = oshape[1], pooled_height = oshape[2], pooled_width = oshape[3]; + const int kernel_d = kernel[0], kernel_h = kernel[1], kernel_w = kernel[2]; + const int pad_d = pad[0], pad_h = pad[1], pad_w = pad[2]; + const int stride_d = stride[0], stride_h = stride[1], stride_w = stride[2]; + const int features = oshape[4]; + const index_t in_data_offset = ishape[1] * ishape[2] * ishape[3] * features; + const index_t out_data_offset = oshape[1] * oshape[2] * oshape[3] * features; + std::vector sums(features); + for (index_t n = 0; n < oshape[0]; ++n) { + for (int pd = 0; pd < pooled_depth; ++pd) { + for (int ph = 0; ph < pooled_height; ++ph) { + for (int pw = 0; pw < pooled_width; ++pw) { + int dstart = pd * stride_d - pad_d; + int hstart = ph * stride_h - pad_h; + int wstart = pw * stride_w - pad_w; + int dend = std::min(dstart + kernel_d, depth + pad_d); + int hend = std::min(hstart + kernel_h, height + pad_h); + int wend = std::min(wstart + kernel_w, width + pad_w); + int pool_size = (get_avg ? (dend - dstart) * (hend - hstart) * (wend - wstart) : 1); + dstart = std::max(dstart, 0); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + dend = std::min(dend, depth); + hend = std::min(hend, height); + wend = std::min(wend, width); + if (get_avg && !count_include_pad) { + pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); + } + const int pool_index = (pd * pooled_height + ph) * pooled_width + pw; + std::fill(sums.begin(), sums.end(), 0); + for (int d = dstart; d < dend; ++d) { + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + const int in_index = (d * height + h) * width + w; + for (index_t c = 0; c < features; ++c) { + sums[c] += a_pow_p::Map(in_data[in_index * features + c]) / pool_size; + } + } + } + } + for (index_t c = 0; c < features; ++c) + out_data[pool_index * features + c] = (pool_size == 0) ? + AccType(nanf("")) : + a_root_p::Map(sums[c]); + } + } + } + in_data += in_data_offset; + out_data += out_data_offset; + } +} + +/*! + * \brief max unpooling cpu function for 1-D images in 'ncw' layout. * Do not call this kernel directly. Use the interface unpool(). */ template -inline void unpool_max_1d_cpu(const DType* out_grad, const DType* in_data, - const DType* out_data, const TShape& ishape, - const TShape& oshape, const TShape& kernel, - const TShape& pad, const TShape& stride, - DType* in_grad) { +inline void unpool_max_1d_ncw_cpu(const DType *out_grad, const DType *in_data, + const DType *out_data, const TShape &ishape, + const TShape &oshape, const TShape &kernel, + const TShape &pad, const TShape &stride, + DType *in_grad) { const int width = ishape[2]; const int pooled_width = oshape[2]; const int kernel_w = kernel[0]; @@ -397,15 +702,63 @@ inline void unpool_max_1d_cpu(const DType* out_grad, const DType* in_data, } /*! - * \brief max unpooling cpu function for 2-D images. + * \brief max unpooling cpu function for 1-D images in 'nwc' layout. * Do not call this kernel directly. Use the interface unpool(). */ template -inline void unpool_max_2d_cpu(const DType* out_grad, const DType* in_data, +inline void unpool_max_1d_nwc_cpu(const DType* out_grad, const DType* in_data, const DType* out_data, const TShape& ishape, const TShape& oshape, const TShape& kernel, const TShape& pad, const TShape& stride, DType* in_grad) { + const int width = ishape[1]; + const int pooled_width = oshape[1]; + const int kernel_w = kernel[0]; + const int pad_w = pad[0]; + const int stride_w = stride[0]; + const int features = oshape[2]; + const index_t in_offset = ishape[1] * features; + const index_t out_offset = oshape[1] * features; + std::vector max_idxs(features); + for (index_t n = 0; n < oshape[0]; ++n) { + for (int pw = 0; pw < pooled_width; ++pw) { + int wstart = pw * stride_w - pad_w; + int wend = std::min(wstart + kernel_w, width); + wstart = std::max(wstart, 0); + std::fill(max_idxs.begin(), max_idxs.end(), -1); + for (index_t c = 0; c < features; ++c) { + for (int w = wstart; w < wend; ++w) { + if (in_data[w * features + c] == out_data[pw * features + c]) { + max_idxs[c] = w; + break; + } + } + } + // In the case where pad > 0 and kernel = 1, for example, + // max_idx can be -1 reaching this step. + for (index_t c = 0; c < features; ++c) { + if (max_idxs[c] >= 0) { + in_grad[max_idxs[c] * features + c] += out_grad[pw * features + c]; + } + } + } + in_data += in_offset; + in_grad += in_offset; + out_data += out_offset; + out_grad += out_offset; + } +} + +/*! + * \brief max unpooling cpu function for 2-D images in 'nchw' layout. + * Do not call this kernel directly. Use the interface unpool(). + */ +template +inline void unpool_max_2d_nchw_cpu(const DType *out_grad, const DType *in_data, + const DType *out_data, const TShape &ishape, + const TShape &oshape, const TShape &kernel, + const TShape &pad, const TShape &stride, + DType *in_grad) { const int height = ishape[2], width = ishape[3]; const int pooled_height = oshape[2], pooled_width = oshape[3]; const int kernel_h = kernel[0], kernel_w = kernel[1]; @@ -453,15 +806,75 @@ inline void unpool_max_2d_cpu(const DType* out_grad, const DType* in_data, } /*! - * \brief max unpooling cpu function for 3-D images. + * \brief max unpooling cpu function for 2-D images in 'nhwc' layout. * Do not call this kernel directly. Use the interface unpool(). */ template -inline void unpool_max_3d_cpu(const DType* out_grad, const DType* in_data, +inline void unpool_max_2d_nhwc_cpu(const DType* out_grad, const DType* in_data, const DType* out_data, const TShape& ishape, const TShape& oshape, const TShape& kernel, const TShape& pad, const TShape& stride, DType* in_grad) { + const int height = ishape[1], width = ishape[2]; + const int pooled_height = oshape[1], pooled_width = oshape[2]; + const int kernel_h = kernel[0], kernel_w = kernel[1]; + const int pad_h = pad[0], pad_w = pad[1]; + const int stride_h = stride[0], stride_w = stride[1]; + const int features = oshape[3]; + const index_t in_offset = ishape[1] * ishape[2] * features; + const index_t out_offset = oshape[1] * oshape[2] * features; + std::vector max_idxs(features); + for (index_t n = 0; n < oshape[0]; ++n) { + for (int ph = 0; ph < pooled_height; ++ph) { + for (int pw = 0; pw < pooled_width; ++pw) { + int hstart = ph * stride_h - pad_h; + int wstart = pw * stride_w - pad_w; + int hend = std::min(hstart + kernel_h, height); + int wend = std::min(wstart + kernel_w, width); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + const int pool_index = ph * pooled_width + pw; + std::fill(max_idxs.begin(), max_idxs.end(), -1); + for (index_t c = 0; c < features; ++c) { + bool found = false; + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + const int idx = h * width + w; + if (in_data[idx * features + c] == out_data[pool_index * features + c]) { + max_idxs[c] = idx; + found = true; + break; + } + } + if (found) break; + } + } + // In the case where pad > 0 and kernel = 1, for example, + // max_idx can be -1 reaching this step. + for (index_t c = 0; c < features; ++c) { + if (max_idxs[c] >= 0) { + in_grad[max_idxs[c] * features + c] += out_grad[pool_index * features + c]; + } + } + } + } + in_data += in_offset; + in_grad += in_offset; + out_data += out_offset; + out_grad += out_offset; + } +} + +/*! + * \brief max unpooling cpu function for 3-D images in 'ncdhw' layout. + * Do not call this kernel directly. Use the interface unpool(). + */ +template +inline void unpool_max_3d_ncdhw_cpu(const DType *out_grad, const DType *in_data, + const DType *out_data, const TShape &ishape, + const TShape &oshape, const TShape &kernel, + const TShape &pad, const TShape &stride, + DType *in_grad) { const int depth = ishape[2], height = ishape[3], width = ishape[4]; const int pooled_depth = oshape[2], pooled_height = oshape[3], pooled_width = oshape[4]; const int kernel_d = kernel[0], kernel_h = kernel[1], kernel_w = kernel[2]; @@ -517,14 +930,83 @@ inline void unpool_max_3d_cpu(const DType* out_grad, const DType* in_data, } /*! - * \brief avg/sum unpooling cpu function for 1-D images. + * \brief max unpooling cpu function for 3-D images in 'ndhwc' layout. + * Do not call this kernel directly. Use the interface unpool(). + */ +template +inline void unpool_max_3d_ndhwc_cpu(const DType* out_grad, const DType* in_data, + const DType* out_data, const TShape& ishape, + const TShape& oshape, const TShape& kernel, + const TShape& pad, const TShape& stride, + DType* in_grad) { + const int depth = ishape[1], height = ishape[2], width = ishape[3]; + const int pooled_depth = oshape[1], pooled_height = oshape[2], pooled_width = oshape[3]; + const int kernel_d = kernel[0], kernel_h = kernel[1], kernel_w = kernel[2]; + const int pad_d = pad[0], pad_h = pad[1], pad_w = pad[2]; + const int stride_d = stride[0], stride_h = stride[1], stride_w = stride[2]; + const int features = oshape[4]; + const index_t in_offset = ishape[1] * ishape[2] * ishape[3] * features; + const index_t out_offset = oshape[1] * oshape[2] * oshape[3] * features; + std::vector max_idxs(features); + for (index_t n = 0; n < oshape[0]; ++n) { + for (int pd = 0; pd < pooled_depth; ++pd) { + for (int ph = 0; ph < pooled_height; ++ph) { + for (int pw = 0; pw < pooled_width; ++pw) { + int dstart = pd * stride_d - pad_d; + int hstart = ph * stride_h - pad_h; + int wstart = pw * stride_w - pad_w; + int dend = std::min(dstart + kernel_d, depth); + int hend = std::min(hstart + kernel_h, height); + int wend = std::min(wstart + kernel_w, width); + dstart = std::max(dstart, 0); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + const int pool_index = (pd * pooled_height + ph) * pooled_width + pw; + std::fill(max_idxs.begin(), max_idxs.end(), -1); + for (index_t c = 0; c < features; ++c) { + bool found = false; + for (int d = dstart; d < dend; ++d) { + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + const int idx = (d * height + h) * width + w; + if (in_data[idx * features + c] == out_data[pool_index * features + c]) { + max_idxs[c] = idx; + found = true; + break; + } + } + if (found) break; + } + if (found) break; + } + } + // In the case where pad > 0 and kernel = 1, for example, + // max_idx can be -1 reaching this step. + for (index_t c = 0; c < features; ++c) { + if (max_idxs[c] >= 0) { + in_grad[max_idxs[c] * features + c] += out_grad[pool_index * features + c]; + } + } + } + } + } + in_data += in_offset; + in_grad += in_offset; + out_data += out_offset; + out_grad += out_offset; + } +} + +/*! + * \brief avg/sum unpooling cpu function for 1-D images in 'ncw' layout. * Do not call this kernel directly. Use the interface unpool(). */ template -inline void unpool_sum_1d_cpu(const DType* out_grad, const DType* in_data, const DType* out_data, - const TShape& ishape, const TShape& oshape, const TShape& kernel, - const TShape& pad, const TShape& stride, DType* in_grad, - const bool is_avg = false, const bool count_include_pad = true) { +inline void unpool_sum_1d_ncw_cpu(const DType *out_grad, const DType *in_data, + const DType *out_data, + const TShape &ishape, const TShape &oshape, const TShape &kernel, + const TShape &pad, const TShape &stride, DType *in_grad, + const bool is_avg = false, const bool count_include_pad = true) { const int width = ishape[2]; const int pooled_width = oshape[2]; const int kernel_w = kernel[0]; @@ -556,14 +1038,61 @@ inline void unpool_sum_1d_cpu(const DType* out_grad, const DType* in_data, const } /*! - * \brief avg/sum unpooling cpu function for 2-D images. + * \brief avg/sum unpooling cpu function for 1-D images in 'nwc' layout. + * Do not call this kernel directly. Use the interface unpool(). + */ +template +inline void unpool_sum_1d_nwc_cpu(const DType* out_grad, const DType* in_data, + const DType *out_data, const TShape &ishape, + const TShape &oshape, const TShape &kernel, + const TShape &pad, const TShape &stride, + DType *in_grad, const bool is_avg = false, + const bool count_include_pad = true) { + const int width = ishape[1]; + const int pooled_width = oshape[1]; + const int kernel_w = kernel[0]; + const int pad_w = pad[0]; + const int stride_w = stride[0]; + const int features = oshape[2]; + const index_t in_grad_offset = ishape[1] * features; + const index_t out_grad_offset = oshape[1] * features; + for (index_t n = 0; n < oshape[0]; ++n) { + for (int pw = 0; pw < pooled_width; ++pw) { + int wstart = pw * stride_w - pad_w; + int wend = std::min(wstart + kernel_w, width + pad_w); + int pool_size = (is_avg ? (wend - wstart) : 1); + wstart = std::max(wstart, 0); + wend = std::min(wend, width); + if (is_avg && !count_include_pad) { + pool_size = (wend - wstart); + } + for (int w = wstart; w < wend; ++w) { + for (index_t c = 0; c < features; ++c) { + in_grad[w * features + c] += + lp_grad::Map(out_grad[pw * features + c], + in_data[w * features + c], + out_data[pw * features + c]) / pool_size; + } + } + } + in_grad += in_grad_offset; + in_data += in_grad_offset; + out_grad += out_grad_offset; + out_data += out_grad_offset; + } +} + +/*! + * \brief avg/sum unpooling cpu function for 2-D images in 'nchw' layout. * Do not call this kernel directly. Use the interface unpool(). */ template -inline void unpool_sum_2d_cpu(const DType* out_grad, const DType* in_data, const DType* out_data, - const TShape& ishape, const TShape& oshape, const TShape& kernel, - const TShape& pad, const TShape& stride, DType* in_grad, - const bool is_avg = false, const bool count_include_pad = true) { +inline void unpool_sum_2d_nchw_cpu(const DType *out_grad, const DType *in_data, + const DType *out_data, const TShape &ishape, + const TShape &oshape, const TShape &kernel, + const TShape &pad, const TShape &stride, + DType *in_grad, const bool is_avg = false, + const bool count_include_pad = true) { const int height = ishape[2], width = ishape[3]; const int pooled_height = oshape[2], pooled_width = oshape[3]; const int kernel_h = kernel[0], kernel_w = kernel[1]; @@ -607,14 +1136,71 @@ inline void unpool_sum_2d_cpu(const DType* out_grad, const DType* in_data, const } /*! - * \brief avg/sum unpooling cpu function for 3-D images. + * \brief avg/sum unpooling cpu function for 2-D images in 'nhwc' layout. * Do not call this kernel directly. Use the interface unpool(). */ template -inline void unpool_sum_3d_cpu(const DType* out_grad, const DType* in_data, const DType* out_data, - const TShape& ishape, const TShape& oshape, const TShape& kernel, - const TShape& pad, const TShape& stride, DType* in_grad, - const bool is_avg = false, const bool count_include_pad = true) { +inline void unpool_sum_2d_nhwc_cpu(const DType* out_grad, const DType* in_data, + const DType *out_data, const TShape &ishape, + const TShape &oshape, const TShape &kernel, + const TShape &pad, const TShape &stride, + DType *in_grad, const bool is_avg = false, + const bool count_include_pad = true) { + const int height = ishape[1], width = ishape[2]; + const int pooled_height = oshape[1], pooled_width = oshape[2]; + const int kernel_h = kernel[0], kernel_w = kernel[1]; + const int pad_h = pad[0], pad_w = pad[1]; + const int features = oshape[3]; + const int stride_h = stride[0], stride_w = stride[1]; + const index_t in_grad_offset = ishape[1] * ishape[2] * features; + const index_t out_grad_offset = oshape[1] * oshape[2] * features; + for (index_t n = 0; n < oshape[0]; ++n) { + for (int ph = 0; ph < pooled_height; ++ph) { + for (int pw = 0; pw < pooled_width; ++pw) { + int hstart = ph * stride_h - pad_h; + int wstart = pw * stride_w - pad_w; + int hend = std::min(hstart + kernel_h, height + pad_h); + int wend = std::min(wstart + kernel_w, width + pad_w); + int pool_size = (is_avg ? (hend - hstart) * (wend - wstart) : 1); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + hend = std::min(hend, height); + wend = std::min(wend, width); + if (is_avg && !count_include_pad) { + pool_size = (hend - hstart) * (wend - wstart); + } + const int pool_index = ph * pooled_width + pw; + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + const int in_index = h * width + w; + for (index_t c = 0; c < features; ++c) { + in_grad[in_index * features + c] += + lp_grad::Map(out_grad[pool_index * features + c], + in_data[in_index * features + c], + out_data[pool_index * features + c]) / pool_size; + } + } + } + } + } + in_grad += in_grad_offset; + in_data += in_grad_offset; + out_grad += out_grad_offset; + out_data += out_grad_offset; + } +} + +/*! + * \brief avg/sum unpooling cpu function for 3-D images in 'ncdhw' layout. + * Do not call this kernel directly. Use the interface unpool(). + */ +template +inline void unpool_sum_3d_ncdhw_cpu(const DType *out_grad, const DType *in_data, + const DType *out_data, const TShape &ishape, + const TShape &oshape, const TShape &kernel, + const TShape &pad, const TShape &stride, + DType *in_grad, const bool is_avg = false, + const bool count_include_pad = true) { const int depth = ishape[2], height = ishape[3], width = ishape[4]; const int pooled_depth = oshape[2], pooled_height = oshape[3], pooled_width = oshape[4]; const int kernel_d = kernel[0], kernel_h = kernel[1], kernel_w = kernel[2]; @@ -665,6 +1251,69 @@ inline void unpool_sum_3d_cpu(const DType* out_grad, const DType* in_data, const } } +/*! + * \brief avg/sum unpooling cpu function for 3-D images in 'ndhwc' layout. + * Do not call this kernel directly. Use the interface unpool(). + */ +template +inline void unpool_sum_3d_ndhwc_cpu(const DType* out_grad, const DType* in_data, + const DType *out_data, const TShape &ishape, + const TShape &oshape, const TShape &kernel, + const TShape &pad, const TShape &stride, + DType *in_grad, const bool is_avg = false, + const bool count_include_pad = true) { + const int depth = ishape[1], height = ishape[2], width = ishape[3]; + const int pooled_depth = oshape[1], pooled_height = oshape[2], pooled_width = oshape[3]; + const int kernel_d = kernel[0], kernel_h = kernel[1], kernel_w = kernel[2]; + const int pad_d = pad[0], pad_h = pad[1], pad_w = pad[2]; + const int stride_d = stride[0], stride_h = stride[1], stride_w = stride[2]; + const int features = oshape[4]; + const index_t in_grad_offset = ishape[1] * ishape[2] * ishape[3] * features; + const index_t out_grad_offset = oshape[1] * oshape[2] * oshape[3] * features; + for (index_t n = 0; n < oshape[0]; ++n) { + for (int pd = 0; pd < pooled_depth; ++pd) { + for (int ph = 0; ph < pooled_height; ++ph) { + for (int pw = 0; pw < pooled_width; ++pw) { + int dstart = pd * stride_d - pad_d; + int hstart = ph * stride_h - pad_h; + int wstart = pw * stride_w - pad_w; + int dend = std::min(dstart + kernel_d, depth + pad_d); + int hend = std::min(hstart + kernel_h, height + pad_h); + int wend = std::min(wstart + kernel_w, width + pad_w); + int pool_size = (is_avg ? (dend - dstart) * (hend - hstart) * (wend - wstart) : 1); + dstart = std::max(dstart, 0); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + dend = std::min(dend, depth); + hend = std::min(hend, height); + wend = std::min(wend, width); + if (is_avg && !count_include_pad) { + pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); + } + const int pool_index = (pd * pooled_height + ph) * pooled_width + pw; + for (int d = dstart; d < dend; ++d) { + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + const int in_index = (d * height + h) * width + w; + for (index_t c = 0; c < features; ++c) { + in_grad[in_index * features + c] += + lp_grad::Map(out_grad[pool_index * features + c], + in_data[in_index * features + c], + out_data[pool_index * features + c]) / pool_size; + } + } + } + } + } + } + } + in_grad += in_grad_offset; + in_data += in_grad_offset; + out_grad += out_grad_offset; + out_data += out_grad_offset; + } +} + /*! * \brief This function serves as an interface for 1/2/3-D pooling operations. * \param s context stream defining the device in use is cpu @@ -683,46 +1332,97 @@ template inline void pool(mshadow::Stream* s, const DType* in_data, const TShape& ishape, const TShape& oshape, const TShape& kernel, const TShape& pad, const TShape& stride, const int pool_type, OpReqType req_type, - DType* out_data, const bool count_include_pad) { + DType* out_data, const bool count_include_pad, int layout) { CHECK_EQ(req_type, kWriteTo) << "Only support req=kWriteTo in pooling operations"; if (kernel.ndim() == 1) { - if (pool_enum::kMaxPooling == pool_type) { - pool_max_1d_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); - } else if (pool_enum::kAvgPooling == pool_type) { - pool_sum_1d_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data, - true, count_include_pad); - } else if (pool_enum::kSumPooling == pool_type) { - pool_sum_1d_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); - } else if (pool_enum::kLpPooling == pool_type) { - pool_sum_1d_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + if (layout == mshadow::kNWC) { + if (pool_enum::kMaxPooling == pool_type) { + pool_max_1d_nwc_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else if (pool_enum::kAvgPooling == pool_type) { + pool_sum_1d_nwc_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data, + true, count_include_pad); + } else if (pool_enum::kSumPooling == pool_type) { + pool_sum_1d_nwc_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else if (pool_enum::kLpPooling == pool_type) { + pool_sum_1d_nwc_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else { + LOG(FATAL) << "Unknown pooling type " << pool_type; + } + } else if (layout == mshadow::kNCW) { + if (pool_enum::kMaxPooling == pool_type) { + pool_max_1d_ncw_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else if (pool_enum::kAvgPooling == pool_type) { + pool_sum_1d_ncw_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data, + true, count_include_pad); + } else if (pool_enum::kSumPooling == pool_type) { + pool_sum_1d_ncw_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else if (pool_enum::kLpPooling == pool_type) { + pool_sum_1d_ncw_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else { + LOG(FATAL) << "Unknown pooling type " << pool_type; + } } else { - LOG(FATAL) << "Unknown pooling type " << pool_type; + LOG(FATAL) << "Unsupported layout, expecting kNCW or kNWC, saw: " << layout; } } else if (kernel.ndim() == 2) { - if (pool_enum::kMaxPooling == pool_type) { - pool_max_2d_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); - } else if (pool_enum::kAvgPooling == pool_type) { - pool_sum_2d_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data, - true, count_include_pad); - } else if (pool_enum::kSumPooling == pool_type) { - pool_sum_2d_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); - } else if (pool_enum::kLpPooling == pool_type) { - pool_sum_2d_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + if (layout == mshadow::kNHWC) { + if (pool_enum::kMaxPooling == pool_type) { + pool_max_2d_nhwc_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else if (pool_enum::kAvgPooling == pool_type) { + pool_sum_2d_nhwc_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data, + true, count_include_pad); + } else if (pool_enum::kSumPooling == pool_type) { + pool_sum_2d_nhwc_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else if (pool_enum::kLpPooling == pool_type) { + pool_sum_2d_nhwc_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else { + LOG(FATAL) << "Unknown pooling type " << pool_type; + } + } else if (layout == mshadow::kNCHW) { + if (pool_enum::kMaxPooling == pool_type) { + pool_max_2d_nchw_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else if (pool_enum::kAvgPooling == pool_type) { + pool_sum_2d_nchw_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data, + true, count_include_pad); + } else if (pool_enum::kSumPooling == pool_type) { + pool_sum_2d_nchw_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else if (pool_enum::kLpPooling == pool_type) { + pool_sum_2d_nchw_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else { + LOG(FATAL) << "Unknown pooling type " << pool_type; + } } else { - LOG(FATAL) << "Unknown pooling type " << pool_type; + LOG(FATAL) << "Unsupported layout, expecting kNCHW or kNHWC, saw: " << layout; } } else if (kernel.ndim() == 3) { - if (pool_enum::kMaxPooling == pool_type) { - pool_max_3d_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); - } else if (pool_enum::kAvgPooling == pool_type) { - pool_sum_3d_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data, - true, count_include_pad); - } else if (pool_enum::kSumPooling == pool_type) { - pool_sum_3d_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); - } else if (pool_enum::kLpPooling == pool_type) { - pool_sum_3d_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + if (layout == mshadow::kNDHWC) { + if (pool_enum::kMaxPooling == pool_type) { + pool_max_3d_ndhwc_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else if (pool_enum::kAvgPooling == pool_type) { + pool_sum_3d_ndhwc_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data, + true, count_include_pad); + } else if (pool_enum::kSumPooling == pool_type) { + pool_sum_3d_ndhwc_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else if (pool_enum::kLpPooling == pool_type) { + pool_sum_3d_ndhwc_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else { + LOG(FATAL) << "Unknown pooling type " << pool_type; + } + } else if (layout == mshadow::kNCDHW) { + if (pool_enum::kMaxPooling == pool_type) { + pool_max_3d_ncdhw_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else if (pool_enum::kAvgPooling == pool_type) { + pool_sum_3d_ncdhw_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data, + true, count_include_pad); + } else if (pool_enum::kSumPooling == pool_type) { + pool_sum_3d_ncdhw_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else if (pool_enum::kLpPooling == pool_type) { + pool_sum_3d_ncdhw_cpu(in_data, ishape, oshape, kernel, pad, stride, out_data); + } else { + LOG(FATAL) << "Unknown pooling type " << pool_type; + } } else { - LOG(FATAL) << "Unknown pooling type " << pool_type; + LOG(FATAL) << "Unsupported layout, expecting kNCDHW or kNDHWC, saw: " << layout; } } else { LOG(FATAL) << "Unsupported " << kernel.ndim() << "-D pooling"; @@ -750,52 +1450,128 @@ inline void unpool(mshadow::Stream* s, const DType* out_grad, const DType* const DType* out_data, const TShape& ishape, const TShape& oshape, const TShape& kernel, const TShape& pad, const TShape& stride, const int pool_type, OpReqType req_type, DType* in_grad, - const bool count_include_pad) { + const bool count_include_pad, int layout) { if (mxnet::kNullOp == req_type) return; if (mxnet::kAddTo != req_type) { mxnet_op::Kernel::Launch(s, ishape.Size(), in_grad); } if (kernel.ndim() == 1) { - if (pool_enum::kMaxPooling == pool_type) { - unpool_max_1d_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, in_grad); - } else if (pool_enum::kAvgPooling == pool_type) { - unpool_sum_1d_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, in_grad, - true, count_include_pad); - } else if (pool_enum::kSumPooling == pool_type) { - unpool_sum_1d_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, in_grad); - } else if (pool_enum::kLpPooling == pool_type) { - unpool_sum_1d_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, - in_grad); + if (layout == mshadow::kNWC) { + if (pool_enum::kMaxPooling == pool_type) { + unpool_max_1d_nwc_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad); + } else if (pool_enum::kAvgPooling == pool_type) { + unpool_sum_1d_nwc_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad, true, count_include_pad); + } else if (pool_enum::kSumPooling == pool_type) { + unpool_sum_1d_nwc_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad); + } else if (pool_enum::kLpPooling == pool_type) { + unpool_sum_1d_nwc_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, + stride, + in_grad); + } else { + LOG(FATAL) << "Unknown pooling type " << pool_type; + } + } else if (layout == mshadow::kNCW) { + if (pool_enum::kMaxPooling == pool_type) { + unpool_max_1d_ncw_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad); + } else if (pool_enum::kAvgPooling == pool_type) { + unpool_sum_1d_ncw_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad, + true, count_include_pad); + } else if (pool_enum::kSumPooling == pool_type) { + unpool_sum_1d_ncw_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad); + } else if (pool_enum::kLpPooling == pool_type) { + unpool_sum_1d_ncw_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, + stride, + in_grad); + } else { + LOG(FATAL) << "Unknown pooling type " << pool_type; + } } else { - LOG(FATAL) << "Unknown pooling type " << pool_type; + LOG(FATAL) << "Unsupported layout, expecting kNCW or kNWC, saw: " << layout; } } else if (kernel.ndim() == 2) { - if (pool_enum::kMaxPooling == pool_type) { - unpool_max_2d_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, in_grad); - } else if (pool_enum::kAvgPooling == pool_type) { - unpool_sum_2d_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, in_grad, - true, count_include_pad); - } else if (pool_enum::kSumPooling == pool_type) { - unpool_sum_2d_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, in_grad); - } else if (pool_enum::kLpPooling == pool_type) { - unpool_sum_2d_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, - in_grad); + if (layout == mshadow::kNHWC) { + if (pool_enum::kMaxPooling == pool_type) { + unpool_max_2d_nhwc_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad); + } else if (pool_enum::kAvgPooling == pool_type) { + unpool_sum_2d_nhwc_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad, + true, count_include_pad); + } else if (pool_enum::kSumPooling == pool_type) { + unpool_sum_2d_nhwc_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad); + } else if (pool_enum::kLpPooling == pool_type) { + unpool_sum_2d_nhwc_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, + stride, + in_grad); + } else { + LOG(FATAL) << "Unknown pooling type " << pool_type; + } + } else if (layout == mshadow::kNCHW) { + if (pool_enum::kMaxPooling == pool_type) { + unpool_max_2d_nchw_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad); + } else if (pool_enum::kAvgPooling == pool_type) { + unpool_sum_2d_nchw_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad, + true, count_include_pad); + } else if (pool_enum::kSumPooling == pool_type) { + unpool_sum_2d_nchw_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad); + } else if (pool_enum::kLpPooling == pool_type) { + unpool_sum_2d_nchw_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, + stride, + in_grad); + } else { + LOG(FATAL) << "Unknown pooling type " << pool_type; + } } else { - LOG(FATAL) << "Unknown pooling type " << pool_type; + LOG(FATAL) << "Unsupported layout, expecting kNCHW or kNHWC, saw: " << layout; } } else if (kernel.ndim() == 3) { - if (pool_enum::kMaxPooling == pool_type) { - unpool_max_3d_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, in_grad); - } else if (pool_enum::kAvgPooling == pool_type) { - unpool_sum_3d_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, in_grad, - true, count_include_pad); - } else if (pool_enum::kSumPooling == pool_type) { - unpool_sum_3d_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, in_grad); - } else if (pool_enum::kLpPooling == pool_type) { - unpool_sum_3d_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, - in_grad); + if (layout == mshadow::kNDHWC) { + if (pool_enum::kMaxPooling == pool_type) { + unpool_max_3d_ndhwc_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad); + } else if (pool_enum::kAvgPooling == pool_type) { + unpool_sum_3d_ndhwc_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad, true, count_include_pad); + } else if (pool_enum::kSumPooling == pool_type) { + unpool_sum_3d_ndhwc_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad); + } else if (pool_enum::kLpPooling == pool_type) { + unpool_sum_3d_ndhwc_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, + stride, + in_grad); + } else { + LOG(FATAL) << "Unknown pooling type " << pool_type; + } + } else if (layout == mshadow::kNCDHW) { + if (pool_enum::kMaxPooling == pool_type) { + unpool_max_3d_ncdhw_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad); + } else if (pool_enum::kAvgPooling == pool_type) { + unpool_sum_3d_ncdhw_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad, + true, count_include_pad); + } else if (pool_enum::kSumPooling == pool_type) { + unpool_sum_3d_ncdhw_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, stride, + in_grad); + } else if (pool_enum::kLpPooling == pool_type) { + unpool_sum_3d_ncdhw_cpu(out_grad, in_data, out_data, ishape, oshape, kernel, pad, + stride, + in_grad); + } else { + LOG(FATAL) << "Unknown pooling type " << pool_type; + } } else { - LOG(FATAL) << "Unknown pooling type " << pool_type; + LOG(FATAL) << "Unsupported layout, expecting kNCDHW or kNDHWC, saw: " << layout; } } else { LOG(FATAL) << "Unsupported " << kernel.ndim() << "-D unpooling"; diff --git a/src/operator/nn/pool_utils.h b/src/operator/nn/pool_utils.h index 641cc4a995ab..6bf7235048dc 100644 --- a/src/operator/nn/pool_utils.h +++ b/src/operator/nn/pool_utils.h @@ -25,6 +25,17 @@ namespace mxnet { namespace op { +// Define an accumulator type AccType to permit float16-I/O lp pooling to avoid underflow. +template +struct PoolingTypes { + typedef DType AccType; +}; + +template<> +struct PoolingTypes { + typedef float AccType; +}; + template struct a_pow_p { static MSHADOW_XINLINE DType Map(const DType a) { @@ -98,14 +109,17 @@ struct lp_grad { template struct lp_grad { static MSHADOW_XINLINE DType Map(const DType grad, const DType in_data, const DType out_data) { - return grad * in_data / out_data; + // Avoid inf, if out_data has underflowed to 0 for a non-zero input, or nan if grad is also 0. + return (out_data == DType(0.0)) ? DType(0.0) : grad * (in_data / out_data); } }; template struct lp_grad { static MSHADOW_XINLINE DType Map(const DType grad, const DType in_data, const DType out_data) { - return grad * in_data * in_data / (out_data * out_data); + // Avoid inf, if out_data has underflowed to 0 for a non-zero input, or nan if grad is also 0. + DType in_out_ratio = in_data / out_data; + return (out_data == DType(0.0)) ? DType(0.0) : grad * in_out_ratio * in_out_ratio; } }; diff --git a/src/operator/nn/pooling-inl.h b/src/operator/nn/pooling-inl.h index 71d85da9ba52..af00fd5cfa3c 100644 --- a/src/operator/nn/pooling-inl.h +++ b/src/operator/nn/pooling-inl.h @@ -53,6 +53,7 @@ struct PoolingParam : public dmlc::Parameter { bool cudnn_off; dmlc::optional p_value; dmlc::optional count_include_pad; + dmlc::optional layout; DMLC_DECLARE_PARAMETER(PoolingParam) { DMLC_DECLARE_FIELD(kernel).set_default(TShape()) // add default value here .enforce_nonzero() @@ -92,6 +93,17 @@ struct PoolingParam : public dmlc::Parameter { "calculation. For example, with a 5*5 kernel on a 3*3 corner of a image," "the sum of the 9 valid elements will be divided by 25 if this is set to true," "or it will be divided by 9 if this is set to false. Defaults to true."); + + DMLC_DECLARE_FIELD(layout) + .add_enum("NCW", mshadow::kNCW) + .add_enum("NCHW", mshadow::kNCHW) + .add_enum("NCDHW", mshadow::kNCDHW) + .add_enum("NWC", mshadow::kNWC) + .add_enum("NHWC", mshadow::kNHWC) + .add_enum("NDHWC", mshadow::kNDHWC) + .set_default(dmlc::optional()) + .describe("Set layout for input and output. Empty for\n " + "default layout: NCW for 1d, NCHW for 2d and NCDHW for 3d."); } bool operator==(const PoolingParam& other) const { @@ -103,7 +115,28 @@ struct PoolingParam : public dmlc::Parameter { this->global_pool == other.global_pool && this->cudnn_off == other.cudnn_off && this->p_value == other.p_value && - this->count_include_pad == other.count_include_pad; + this->count_include_pad == other.count_include_pad && + this->layout == other.layout; + } + + // Extract layout from param, or supply default layout based on provided input dimension. + int GetLayout(int input_dim) const { + int ret_val = mshadow::kNCW; + if (layout.has_value()) { + ret_val = layout.value(); + } else { + switch (input_dim) { + case 3U: ret_val = mshadow::kNCW; break; + case 4U: ret_val = mshadow::kNCHW; break; + case 5U: ret_val = mshadow::kNCDHW; break; + default: + LOG(FATAL) << "Unexpected input data dim " << input_dim << "\n" + << "Pooling: Input data should be 3D in (batch, channel, x), " + << " or 4D in (batch, channel, y, x), " + << " or 5D in (batch, channel, d, y, x)."; + } + } + return ret_val; } }; @@ -124,6 +157,8 @@ struct hash { ret = dmlc::HashCombine(ret, val.cudnn_off); ret = dmlc::HashCombine(ret, val.p_value); ret = dmlc::HashCombine(ret, val.count_include_pad); + int val_layout = val.layout.has_value() ? val.layout.value() : -1; + ret = dmlc::HashCombine(ret, val_layout); return ret; } }; @@ -154,9 +189,17 @@ class PoolingOp { TShape kernel = param_.kernel; TShape padding = param_.pad; TShape stride = param_.stride; + int layout = param_.GetLayout(ishape.ndim()); if (param_.global_pool) { - kernel = TShape(ishape.data() + 2, - ishape.data() + ishape.ndim()); + // with global pooling, kernel shape corresponds to input shape with 'N' and 'C' removed + if (layout == mshadow::kNWC || layout == mshadow::kNHWC || layout == mshadow::kNDHWC) { + kernel = TShape(ishape.data() + 1, + ishape.data() + ishape.ndim() - 1); + + } else { + kernel = TShape(ishape.data() + 2, + ishape.data() + ishape.ndim()); + } padding = TShape(ishape.ndim() - 2); for (index_t i = 0; i < ishape.ndim() - 2; i++) { padding[i] = 0; @@ -173,21 +216,21 @@ class PoolingOp { kernel, padding, stride, - param_.pool_type, req, out_data.dptr(), count_include_pad); + param_.pool_type, req, out_data.dptr(), count_include_pad, layout); break; case 2: pool(s, in_data.dptr(), in_data.shape_, out_data.shape_, kernel, padding, stride, - param_.pool_type, req, out_data.dptr(), count_include_pad); + param_.pool_type, req, out_data.dptr(), count_include_pad, layout); break; case 3: pool(s, in_data.dptr(), in_data.shape_, out_data.shape_, kernel, padding, stride, - param_.pool_type, req, out_data.dptr(), count_include_pad); + param_.pool_type, req, out_data.dptr(), count_include_pad, layout); break; default: LOG(FATAL) << "p value of " << p_value << " is not supported yet..."; @@ -203,9 +246,17 @@ class PoolingOp { TShape kernel = param_.kernel; TShape padding = param_.pad; TShape stride = param_.stride; + int layout = param_.GetLayout(ishape.ndim()); if (param_.global_pool) { - kernel = TShape(ishape.data() + 2, - ishape.data() + ishape.ndim()); + // with global pooling, kernel shape corresponds to input shape with 'N' and 'C' removed + if (layout == mshadow::kNWC || layout == mshadow::kNHWC || layout == mshadow::kNDHWC) { + kernel = TShape(ishape.data() + 1, + ishape.data() + ishape.ndim() - 1); + + } else { + kernel = TShape(ishape.data() + 2, + ishape.data() + ishape.ndim()); + } padding = TShape(ishape.ndim() - 2); for (index_t i = 0; i < ishape.ndim() - 2; i++) { padding[i] = 0; @@ -224,7 +275,7 @@ class PoolingOp { kernel, padding, stride, - param_.pool_type, req, in_grad.dptr(), count_include_pad); + param_.pool_type, req, in_grad.dptr(), count_include_pad, layout); break; case 2: unpool(s, out_grad.dptr(), in_data.dptr(), out_data.dptr(), @@ -232,7 +283,7 @@ class PoolingOp { kernel, padding, stride, - param_.pool_type, req, in_grad.dptr(), count_include_pad); + param_.pool_type, req, in_grad.dptr(), count_include_pad, layout); break; case 3: unpool(s, out_grad.dptr(), in_data.dptr(), out_data.dptr(), @@ -240,7 +291,7 @@ class PoolingOp { kernel, padding, stride, - param_.pool_type, req, in_grad.dptr(), count_include_pad); + param_.pool_type, req, in_grad.dptr(), count_include_pad, layout); break; default: LOG(FATAL) << "p value of " << p_value << " is not supported yet..."; diff --git a/src/operator/nn/pooling.cc b/src/operator/nn/pooling.cc index 611568807a9a..9e9af4d97fd9 100644 --- a/src/operator/nn/pooling.cc +++ b/src/operator/nn/pooling.cc @@ -39,6 +39,9 @@ void PoolingParamParser(nnvm::NodeAttrs *attrs) { using namespace mshadow; PoolingParam param; param.Init(attrs->dict); + // Set default layout if it can be inferred from kernel shape. + if (param.kernel.ndim() > 0) + param.layout = param.GetLayout(param.kernel.ndim() + 2); if (param.kernel.ndim() == 1) { if (param.stride.ndim() == 0) param.stride = Shape1(1); if (param.pad.ndim() == 0) param.pad = Shape1(0); @@ -111,38 +114,65 @@ static bool PoolingShape(const nnvm::NodeAttrs &attrs, << "Pooling: Input data should be 3D in (batch, channel, x)" << " Or 4D in (batch, channel, y, x) " << " Or 5D in (batch, channel, d, y, x)"; - TShape oshape = dshape; if (dshape.ndim() == 0) return false; + int layout = param.GetLayout(dshape.ndim()); if (param.global_pool) { - for (size_t i{2}; i < dshape.ndim(); i++) - oshape[i] = 1; - out_shape->clear(); - out_shape->push_back(oshape); // save output shape + TShape oshape = dshape; + size_t c_index = 0; + switch (layout) { + case mshadow::kNCW: + case mshadow::kNCHW: + case mshadow::kNCDHW: + c_index = 1; + break; + case mshadow::kNWC: + case mshadow::kNHWC: + case mshadow::kNDHWC: + c_index = dshape.ndim() - 1; + break; + default: + LOG(FATAL) << "Unsupported tensor layout " << param.layout.value(); + } + for (size_t i{1}; i < dshape.ndim(); i++) + if (i != c_index) + oshape[i] = 1; + out_shape->clear(); + out_shape->push_back(oshape); // save output shape #if MXNET_USE_MKLDNN == 1 - if (MKLDNNRequireWorkspace(param) && SupportMKLDNNPooling(param)) + if (MKLDNNRequireWorkspace(param) && SupportMKLDNNPooling(param)) out_shape->push_back(oshape); // for workspace #endif + } else if (param.kernel.ndim() == 0) { + return false; } else if (param.kernel.ndim() == 1) { - CHECK_EQ(dshape.ndim(), 3U) - << "Pooling: Input data should be 3D in (batch, channel, x)"; - CHECK(param.kernel[0] <= dshape[2] + 2 * param.pad[0]) - << "kernel size (" << param.kernel[0] << ") exceeds input (" - << dshape[2] << " padded to " << (dshape[2] + 2 * param.pad[0]) - << ")"; + CHECK_EQ(dshape.ndim(), 3U) << + "Pooling: Input data should be 3D in (batch, channel, x)"; + CHECK(layout == mshadow::kNCW || layout == mshadow::kNWC) << "Need 1D layout"; + // Perform shape calculations in a standard (NCW) layout space + mshadow::Shape<3> dshape_ncw = (layout == mshadow::kNWC) ? + ConvertLayout(dshape.get<3>(), mshadow::kNWC, mshadow::kNCW) : + dshape.get<3>(); + mshadow::Shape<3> oshape_ncw = dshape_ncw; + CHECK(param.kernel[0] <= dshape_ncw[2] + 2 * param.pad[0]) + << "kernel size (" << param.kernel[0] << ") exceeds input (" << dshape[2] + << " padded to " << (dshape_ncw[2] + 2*param.pad[0]) << ")"; if (param.pooling_convention == pool_enum::kValid) { - oshape[2] = 1 + - (dshape[2] + 2 * param.pad[0] - param.kernel[0]) / - param.stride[0]; + oshape_ncw[2] = 1 + + (dshape_ncw[2] + 2 * param.pad[0] - param.kernel[0]) / + param.stride[0]; } else if (param.pooling_convention == pool_enum::kFull) { - oshape[2] = 1 + static_cast(std::ceil( - static_cast(dshape[2] + 2 * param.pad[0] - - param.kernel[0]) / - param.stride[0])); + oshape_ncw[2] = 1 + static_cast(std::ceil( + static_cast(dshape_ncw[2] + 2 * param.pad[0] - + param.kernel[0]) / + param.stride[0])); } else { - oshape[2] = static_cast(std::ceil( - static_cast(dshape[2] + 2 * param.pad[0]) / + oshape_ncw[2] = static_cast(std::ceil( + static_cast(dshape_ncw[2] + 2 * param.pad[0]) / param.stride[0])); } + // Convert back from standard (NCW) layout space to the actual layout type + TShape oshape = (layout == mshadow::kNWC) ? + ConvertLayout(oshape_ncw, mshadow::kNCW, mshadow::kNWC) : oshape_ncw; out_shape->clear(); out_shape->push_back(oshape); // save output shape #if MXNET_USE_MKLDNN == 1 @@ -150,33 +180,37 @@ static bool PoolingShape(const nnvm::NodeAttrs &attrs, out_shape->push_back(oshape); // for workspace #endif } else if (param.kernel.ndim() == 2) { - CHECK_EQ(dshape.ndim(), 4U) - << "Pooling: Input data should be 4D in (batch, channel, y, x)"; - CHECK(param.kernel[0] <= dshape[2] + 2 * param.pad[0]) - << "kernel size (" << param.kernel[0] << ") exceeds input (" - << dshape[2] << " padded to " << (dshape[2] + 2 * param.pad[0]) - << ")"; - CHECK(param.kernel[1] <= dshape[3] + 2 * param.pad[1]) - << "kernel size (" << param.kernel[1] << ") exceeds input (" - << dshape[3] << " padded to " << (dshape[3] + 2 * param.pad[1]) - << ")"; + CHECK_EQ(dshape.ndim(), 4U) << "Pooling: Input data should be 4D in (batch, channel, y, x)"; + CHECK(layout == mshadow::kNCHW || layout == mshadow::kNHWC) << "Need 2D layout"; + // Perform shape calculations in a standard (NCHW) layout space + mshadow::Shape<4> dshape_nchw = (layout == mshadow::kNHWC) ? + ConvertLayout(dshape.get<4>(), mshadow::kNHWC, mshadow::kNCHW) : + dshape.get<4>(); + mshadow::Shape<4> oshape_nchw = dshape_nchw; + CHECK(param.kernel[0] <= dshape_nchw[2] + 2 * param.pad[0]) + << "kernel size (" << param.kernel[0] << ") exceeds input (" << dshape_nchw[2] + << " padded to " << (dshape_nchw[2] + 2*param.pad[0]) << ")"; + CHECK(param.kernel[1] <= dshape_nchw[3] + 2 * param.pad[1]) + << "kernel size (" << param.kernel[1] << ") exceeds input (" << dshape_nchw[3] + << " padded to " << (dshape_nchw[3] + 2*param.pad[1]) << ")"; if (param.pooling_convention == pool_enum::kValid) { - oshape[2] = 1 + - (dshape[2] + 2 * param.pad[0] - param.kernel[0]) / - param.stride[0]; - oshape[3] = 1 + - (dshape[3] + 2 * param.pad[1] - param.kernel[1]) / - param.stride[1]; + oshape_nchw[2] = 1 + (dshape_nchw[2] + 2 * param.pad[0] - param.kernel[0]) / + param.stride[0]; + oshape_nchw[3] = 1 + (dshape_nchw[3] + 2 * param.pad[1] - param.kernel[1]) / + param.stride[1]; } else { - oshape[2] = 1 + static_cast(std::ceil( - static_cast(dshape[2] + 2 * param.pad[0] - - param.kernel[0]) / - param.stride[0])); - oshape[3] = 1 + static_cast(std::ceil( - static_cast(dshape[3] + 2 * param.pad[1] - - param.kernel[1]) / - param.stride[1])); + oshape_nchw[2] = 1 + static_cast(ceil( + static_cast(dshape_nchw[2] + 2 * param.pad[0] - + param.kernel[0]) / + param.stride[0])); + oshape_nchw[3] = 1 + static_cast(ceil( + static_cast(dshape_nchw[3] + 2 * param.pad[1] - + param.kernel[1]) / + param.stride[1])); } + // Convert back from standard (NCHW) layout space to the actual layout type + TShape oshape = (layout == mshadow::kNHWC) ? + ConvertLayout(oshape_nchw, mshadow::kNCHW, mshadow::kNHWC) : oshape_nchw; out_shape->clear(); out_shape->push_back(oshape); // save output shape #if MXNET_USE_MKLDNN == 1 @@ -185,38 +219,40 @@ static bool PoolingShape(const nnvm::NodeAttrs &attrs, #endif } else if (param.kernel.ndim() == 3) { CHECK_EQ(dshape.ndim(), 5U) - << "Pooling: Input data should be 5D in (batch, channel, d, y, x)"; - CHECK_LE(param.kernel[0], dshape[2] + 2 * param.pad[0]) - << "kernel size exceeds input"; - CHECK_LE(param.kernel[1], dshape[3] + 2 * param.pad[1]) - << "kernel size exceeds input"; - CHECK_LE(param.kernel[2], dshape[4] + 2 * param.pad[2]) - << "kernel size exceeds input"; + << "Pooling: Input data should be 5D in (batch, channel, d, y, x)"; + CHECK(layout == mshadow::kNCDHW || layout == mshadow::kNDHWC) << "Need 3D layout"; + // Perform shape calculations in a standard (NCDHW) layout space + mshadow::Shape<5> dshape_ncdhw = (layout == mshadow::kNDHWC) ? + ConvertLayout(dshape.get<5>(), mshadow::kNDHWC, mshadow::kNCDHW) : + dshape.get<5>(); + mshadow::Shape<5> oshape_ncdhw = dshape_ncdhw; + CHECK_LE(param.kernel[0], dshape_ncdhw[2] + 2 * param.pad[0]) << "kernel size exceeds input"; + CHECK_LE(param.kernel[1], dshape_ncdhw[3] + 2 * param.pad[1]) << "kernel size exceeds input"; + CHECK_LE(param.kernel[2], dshape_ncdhw[4] + 2 * param.pad[2]) << "kernel size exceeds input"; if (param.pooling_convention == pool_enum::kValid) { - oshape[2] = 1 + - (dshape[2] + 2 * param.pad[0] - param.kernel[0]) / - param.stride[0]; - oshape[3] = 1 + - (dshape[3] + 2 * param.pad[1] - param.kernel[1]) / - param.stride[1]; - oshape[4] = 1 + - (dshape[4] + 2 * param.pad[2] - param.kernel[2]) / - param.stride[2]; + oshape_ncdhw[2] = 1 + (dshape_ncdhw[2] + 2 * param.pad[0] - param.kernel[0]) / + param.stride[0]; + oshape_ncdhw[3] = 1 + (dshape_ncdhw[3] + 2 * param.pad[1] - param.kernel[1]) / + param.stride[1]; + oshape_ncdhw[4] = 1 + (dshape_ncdhw[4] + 2 * param.pad[2] - param.kernel[2]) / + param.stride[2]; } else { - oshape[2] = 1 + static_cast(std::ceil( - static_cast(dshape[2] + 2 * param.pad[0] - - param.kernel[0]) / - param.stride[0])); - oshape[3] = 1 + static_cast(std::ceil( - static_cast(dshape[3] + 2 * param.pad[1] - - param.kernel[1]) / - param.stride[1])); - oshape[4] = 1 + static_cast(std::ceil( - static_cast(dshape[4] + 2 * param.pad[2] - - param.kernel[2]) / - param.stride[2])); + oshape_ncdhw[2] = 1 + static_cast(ceil( + static_cast(dshape_ncdhw[2] + 2 * param.pad[0] - + param.kernel[0]) / + param.stride[0])); + oshape_ncdhw[3] = 1 + static_cast(ceil( + static_cast(dshape_ncdhw[3] + 2 * param.pad[1] - + param.kernel[1]) / + param.stride[1])); + oshape_ncdhw[4] = 1 + static_cast(ceil( + static_cast(dshape_ncdhw[4] + 2 * param.pad[2] - + param.kernel[2]) / + param.stride[2])); } - + // Convert back from standard (NCDHW) layout space to the actual layout type + TShape oshape = (layout == mshadow::kNDHWC) ? + ConvertLayout(oshape_ncdhw, mshadow::kNCDHW, mshadow::kNDHWC) : oshape_ncdhw; out_shape->clear(); out_shape->push_back(oshape); // save output shape #if MXNET_USE_MKLDNN == 1 @@ -224,6 +260,7 @@ static bool PoolingShape(const nnvm::NodeAttrs &attrs, out_shape->push_back(oshape); // for workspace #endif } + return true; } @@ -331,13 +368,13 @@ NNVM_REGISTER_OP(Pooling) The shapes for 1-D pooling are -- **data**: *(batch_size, channel, width)*, -- **out**: *(batch_size, num_filter, out_width)*. +- **data** and **out**: *(batch_size, channel, width)* (NCW layout) or + *(batch_size, width, channel)* (NWC layout), The shapes for 2-D pooling are -- **data**: *(batch_size, channel, height, width)* -- **out**: *(batch_size, num_filter, out_height, out_width)*, with:: +- **data** and **out**: *(batch_size, channel, height, width)* (NCHW layout) or + *(batch_size, height, width, channel)* (NHWC layout), out_height = f(height, kernel[0], pad[0], stride[0]) out_width = f(width, kernel[1], pad[1], stride[1]) @@ -363,8 +400,8 @@ Three pooling options are supported by ``pool_type``: - **lp**: Lp pooling For 3-D pooling, an additional *depth* dimension is added before -*height*. Namely the input data will have shape *(batch_size, channel, depth, -height, width)*. +*height*. Namely the input data and output will have shape *(batch_size, channel, depth, +height, width)* (NCDHW layout) or *(batch_size, depth, height, width, channel)* (NDHWC layout). Notes on Lp pooling: @@ -421,11 +458,13 @@ NNVM_REGISTER_OP(_backward_Pooling) .set_attr( "FInplaceOption", [](const NodeAttrs &attrs) { -#if MXNET_USE_CUDNN == 1 - return std::vector >(); -#else - return std::vector >{{1, 0}}; +// Different backend requires different FInplaceOption +#if MXNET_USE_MKLDNN == 1 + const PoolingParam ¶m = nnvm::get(attrs.parsed); + if (MKLDNNRequireWorkspace(param) && SupportMKLDNNPooling(param)) + return std::vector >{{1, 0}}; #endif + return std::vector >(); }) #if MXNET_USE_MKLDNN == 1 .set_attr("FResourceRequest", [](const NodeAttrs& n) { diff --git a/src/operator/nn/pooling.cu b/src/operator/nn/pooling.cu index 997218620c3a..84cacc15e239 100644 --- a/src/operator/nn/pooling.cu +++ b/src/operator/nn/pooling.cu @@ -56,19 +56,11 @@ void PoolingCompute(const nnvm::NodeAttrs& attrs, CHECK_EQ(outputs.size(), GetNumOutputs(param)); #if MXNET_USE_CUDNN == 1 - if (!param.cudnn_off && param.kernel.ndim() > 1) { + if (!param.cudnn_off) { MSHADOW_REAL_TYPE_SWITCH(inputs[0].type_flag_, DType, { - switch (param.pool_type) { - case pool_enum::kMaxPooling: - case pool_enum::kAvgPooling: - GetCuDNNPoolingOp(param).Forward(ctx, inputs[0], req[0], outputs[0]); - return; - case pool_enum::kSumPooling: - LOG(WARNING) << "Sum pooling is not supported by cudnn, MXNet sum pooling is applied."; - break; - case pool_enum::kLpPooling: - LOG(WARNING) << "Lp pooling is not supported by cudnn, MXNet lp pooling is applied."; - break; + if (CuDNNPoolingOp::Supports(param, inputs[0])) { + GetCuDNNPoolingOp(param).Forward(ctx, inputs[0], req[0], outputs[0]); + return; } }); } @@ -111,21 +103,13 @@ void PoolingGradCompute(const nnvm::NodeAttrs& attrs, } #if MXNET_USE_CUDNN == 1 - if (!param.cudnn_off && param.kernel.ndim() > 1) { + if (!param.cudnn_off) { MSHADOW_REAL_TYPE_SWITCH(inputs[0].type_flag_, DType, { - switch (param.pool_type) { - case pool_enum::kMaxPooling: - case pool_enum::kAvgPooling: + if (CuDNNPoolingOp::Supports(param, inputs[in_data_idx])) { GetCuDNNPoolingOp(param).Backward(ctx, inputs[ograd_idx], inputs[in_data_idx], inputs[out_data_idx], req[0], outputs[0]); return; - case pool_enum::kSumPooling: - LOG(WARNING) << "Sum pooling is not supported by cudnn, MXNet sum pooling is applied."; - break; - case pool_enum::kLpPooling: - LOG(WARNING) << "Lp pooling is not supported by cudnn, MXNet Lp pooling is applied."; - break; } }); } diff --git a/src/operator/quantization/quantized_pooling.cc b/src/operator/quantization/quantized_pooling.cc index 8b62db9c061d..b9daf2592b7d 100644 --- a/src/operator/quantization/quantized_pooling.cc +++ b/src/operator/quantization/quantized_pooling.cc @@ -40,6 +40,9 @@ bool QuantizedPoolingShape(const nnvm::NodeAttrs& attrs, CHECK_EQ(dshape.ndim(), 4U) << "quantized_pooling: Input data should be 4D in " << "(batch, channel, y, x)"; + int layout = param.GetLayout(dshape.ndim()); + CHECK_EQ(layout, mshadow::kNCHW) + << "QuantizedPoolingOp only supports NCHW layout for now, saw " << layout; // NCHW layout const int N = 0, H = 2, W = 3, C = 1; TShape oshape(4); diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 7a7c6f69dd77..010cf504fe70 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -607,7 +607,40 @@ def test_convolution_versions(): check_consistency(syms, ctx_list) +# More max-pooling strides and pads to test cudnn pooling implementation code paths @with_seed() +def test_pooling_nhwc_with_convention(): + def make_pooling_syms(**kwargs): + # Conventional NCHW layout pooling + sym = mx.sym.Pooling(**kwargs) + # NHWC pooling + data = mx.sym.Variable('pool_data') + sym_nhwc = mx.sym.transpose(data, axes=(0,2,3,1)) + sym_nhwc = mx.sym.Pooling(sym_nhwc, layout='NHWC', **kwargs) + sym_nhwc = mx.sym.transpose(sym_nhwc, axes=(0,3,1,2), name='pool') + return [sym, sym_nhwc] + + # While the float32 and float64 output is reliably consistent, float16 departs occasionally. + # We compare nhwc and nchw results only within a given precision. + for in_shape in [(3, 4, 8, 8), (2, 2, 20, 20)]: + for kernel in [(2,2), (3,3), (4,4)]: + for stride in [(1,1), (1,2), (2,1), (2,2)]: + for data_type in [np.float64, np.float32, np.float16]: + ctx_list = [{'ctx': mx.gpu(0), 'pool_data': in_shape, + 'type_dict': {'pool_data': data_type}}] + symlist = make_pooling_syms(kernel=kernel, pool_type='max', stride=stride, + pooling_convention='valid', name='pool') + check_consistency_NxM(symlist, ctx_list) + + symlist = make_pooling_syms(kernel=kernel, pool_type='max', stride=stride, + pooling_convention='full', name='pool') + check_consistency_NxM(symlist, ctx_list) + + symlist = make_pooling_syms(kernel=(300,300), pool_type='max', + global_pool=True, name='pool') + check_consistency_NxM(symlist, ctx_list) + + def test_pooling_with_type(): ctx_list = [{'ctx': mx.gpu(0), 'pool_data': (2, 2, 10, 10), 'type_dict': {'pool_data': np.float64}}, {'ctx': mx.gpu(0), 'pool_data': (2, 2, 10, 10), 'type_dict': {'pool_data': np.float32}}, @@ -768,232 +801,241 @@ def test_spatial_transformer_with_type(): check_consistency(sym, ctx_list) check_consistency(sym, ctx_list, grad_req="add") - @with_seed() def test_pooling_with_type2(): - ctx_list = [{'ctx': mx.gpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float64}}, - {'ctx': mx.gpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float32}}, - {'ctx': mx.gpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float16}}, - {'ctx': mx.cpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float64}}, - {'ctx': mx.cpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float32}}] + # While the float32 and float64 output is reliably consistent, float16 departs occasionally. + # We compare cpu and gpu results only within a given precision. + for data_type in [np.float64, np.float32, np.float16]: + ctx_list = [{'ctx': mx.gpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': data_type}}, + {'ctx': mx.cpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': data_type}}] - sym = mx.sym.Pooling(name='pool', kernel=(3,3), stride=(2,2), pool_type='max') - check_consistency(sym, ctx_list, rand_type=np.float16) + sym = mx.sym.Pooling(name='pool', kernel=(3,3), stride=(2,2), pool_type='max') + check_consistency(sym, ctx_list) - sym = mx.sym.Pooling(name='pool', kernel=(3,3), pad=(1,1), pool_type='avg') - check_consistency(sym, ctx_list) + sym = mx.sym.Pooling(name='pool', kernel=(3,3), pad=(1,1), pool_type='avg') + check_consistency(sym, ctx_list) - sym = mx.sym.Pooling(name='pool', kernel=(5,5), pad=(2,2), pool_type='max') - check_consistency(sym, ctx_list, rand_type=np.float16) + sym = mx.sym.Pooling(name='pool', kernel=(5,5), pad=(2,2), pool_type='max') + check_consistency(sym, ctx_list) + + sym = mx.sym.Pooling(name='pool', kernel=(3,3), pad=(1,1), pool_type='sum') + check_consistency(sym, ctx_list) + +@with_seed() +def test_pooling_nhwc_with_type(): + def make_pooling_syms(**kwargs): + # Conventional NCHW layout pooling + sym = mx.sym.Pooling(**kwargs) + # NHWC pooling + data = mx.sym.Variable('pool_data') + sym_nhwc = mx.sym.transpose(data, axes=(0,2,3,1)) + sym_nhwc = mx.sym.Pooling(sym_nhwc, layout='NHWC', **kwargs) + sym_nhwc = mx.sym.transpose(sym_nhwc, axes=(0,3,1,2), name='pool') + return [sym, sym_nhwc] + + # While the float32 and float64 output is reliably consistent, float16 departs occasionally. + # We compare nhwc and nchw results only within a given precision. + for data_type in [np.float64, np.float32, np.float16]: + # NHWC pooling only enabled on GPU with CUDNN + ctx_list = [{'ctx': mx.gpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': data_type}}] + symlist = make_pooling_syms(name='pool', kernel=(3,3), stride=(2,2), pool_type='max') + check_consistency_NxM(symlist, ctx_list) + + symlist = make_pooling_syms(name='pool', kernel=(3,3), pad=(1,1), pool_type='avg') + check_consistency_NxM(symlist, ctx_list) + + symlist = make_pooling_syms(name='pool', kernel=(5,5), pad=(2,2), pool_type='max') + check_consistency_NxM(symlist, ctx_list) - sym = mx.sym.Pooling(name='pool', kernel=(3,3), pad=(1,1), pool_type='sum') - check_consistency(sym, ctx_list) -@unittest.skip("Flaky test https://github.com/apache/incubator-mxnet/issues/11517") @with_seed() def test_pooling_versions(): - def test_pooling_versions_helper(pool_op_list, data, kernel, pool_type, pad, stride, pooling_convention='valid', - global_pool=False, p_value=2, count_include_pad=True, tol=None): - ctx_list = [] - sym_list = [] - # PoolingV1 cpu - if 'pool_v1_cpu' in pool_op_list: - ctx_list.append({'ctx': mx.cpu(0), 'pool_data': data, 'type_dict': {'pool_data': np.float32}}) - if not global_pool: - sym_list.append(mx.sym.Pooling_v1(kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention=pooling_convention, name='pool')) - else: - sym_list.append(mx.sym.Pooling_v1(kernel=kernel, pool_type=pool_type, global_pool=True, name='pool')) - # PoolingV1 gpu - if 'pool_v1_gpu' in pool_op_list: - ctx_list.append({'ctx': mx.gpu(0), 'pool_data': data, 'type_dict': {'pool_data': np.float32}}) - if not global_pool: - sym_list.append(mx.sym.Pooling_v1(kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention=pooling_convention, name='pool')) - else: - sym_list.append(mx.sym.Pooling_v1(kernel=kernel, pool_type=pool_type, global_pool=True, name='pool')) - # Pooling cpu - if 'pool_cpu' in pool_op_list: - ctx_list.append({'ctx': mx.cpu(0), 'pool_data': data, 'type_dict': {'pool_data': np.float32}}) - if not global_pool: - sym_list.append(mx.sym.Pooling(kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention=pooling_convention, name='pool', - p_value=p_value, count_include_pad=count_include_pad)) - else: - sym_list.append(mx.sym.Pooling(kernel=kernel, pool_type=pool_type, global_pool=True, name='pool', - p_value=p_value, count_include_pad=count_include_pad)) - # Pooling gpu - if 'pool_gpu' in pool_op_list: - ctx_list.append({'ctx': mx.gpu(0), 'pool_data': data, 'type_dict': {'pool_data': np.float32}}) - if not global_pool: - sym_list.append(mx.sym.Pooling(kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention=pooling_convention, cudnn_off=True, name='pool', - p_value=p_value, count_include_pad=count_include_pad)) - else: - sym_list.append(mx.sym.Pooling(kernel=kernel, pool_type=pool_type, global_pool=True, cudnn_off=True, - name='pool', p_value=p_value, count_include_pad=count_include_pad)) - # CuDNNPooling - if 'pool_cudnn' in pool_op_list: - ctx_list.append({'ctx': mx.gpu(0), 'pool_data': data, 'type_dict': {'pool_data': np.float32}}) - if not global_pool: - sym_list.append(mx.sym.Pooling(kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention=pooling_convention, p_value=p_value, cudnn_off=False, - name='pool', count_include_pad=count_include_pad)) - else: - sym_list.append(mx.sym.Pooling(kernel=kernel, pool_type=pool_type, global_pool=True, p_value=p_value, - cudnn_off=False, name='pool', count_include_pad=count_include_pad)) - check_consistency(sym_list, ctx_list, equal_nan=(not count_include_pad), tol=tol) - def test_1d_pooling(pool_type, p_value=2, count_include_pad=True): - data = (2, 3, 20) - kernel = (4,) - pad = (0,) - stride = (1,) - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='valid', global_pool=False, p_value=p_value, - count_include_pad=count_include_pad) + # Produce the name of the 'transposed' layout, given the dimension + def transposed_layout(ndim): + if ndim < 3 or ndim > 5: + raise RuntimeError("Invalid data dim, expecting 3, 4 or 5") + return ('NWC', 'NHWC', 'NDHWC')[ndim-3] - pad = (2,) - stride = (2,) - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='valid', global_pool=False, p_value=p_value, - count_include_pad=count_include_pad) - - pad = (0,) - stride = (1,) - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='full', global_pool=False, p_value=p_value, - count_include_pad=count_include_pad) + # default padding is all zeros + def is_default_pad(pad): + return pad == (0,) * len(pad) - pad = (2,) - stride = (2,) - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='full', global_pool=False, p_value=p_value, - count_include_pad=count_include_pad) + # default stride is all ones + def is_default_stride(stride): + return stride == (1,) * len(stride) - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - global_pool=True, p_value=p_value, count_include_pad=count_include_pad) + # returns True/False randomly with equal probability + def random_choice(): + return np.random.random(1)[0] < 0.5 - def test_2d_pooling(pool_type, p_value=2, count_include_pad=True): - data = (2, 3, 20, 20) - kernel = (4, 5) - pad = (0, 0) - stride = (1, 1) - if pool_type == 'lp': - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='valid', global_pool=False, p_value=p_value) - else: - test_pooling_versions_helper(pool_op_list=['pool_v1_cpu', 'pool_v1_gpu', 'pool_cpu', 'pool_gpu', 'pool_cudnn'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='valid', global_pool=False, count_include_pad=count_include_pad) - - # pool_v1 has bugs when pad is not 0, do not test PoolingV1 here - pad = (2, 3) - stride = (2, 3) - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu', 'pool_cudnn'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='valid', global_pool=False, p_value=p_value, - count_include_pad=count_include_pad) - - pad = (0, 0) - stride = (1, 1) - if pool_type == 'lp': - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu', 'pool_cudnn'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='full', global_pool=False, p_value=p_value) - else: - if count_include_pad: - test_pooling_versions_helper(pool_op_list=['pool_v1_cpu', 'pool_v1_gpu', 'pool_cpu', 'pool_gpu', 'pool_cudnn'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='full', global_pool=False, - count_include_pad=count_include_pad) + def test_pooling_versions_helper(pool_op_list, data, kernel, pool_type, pad, stride, + pooling_convention='valid', global_pool=False, p_value=2, + count_include_pad=True, tol=None, dtype=np.float32): + ctx_list = [] + sym_list = [] + for pool_ctx in pool_op_list: + (pool_op, ctx_type) = pool_ctx.rsplit('_', 1) + expected_ctxs = ['cpu', 'gpu', 'cudnn'] + if ctx_type not in expected_ctxs: + raise RuntimeError('Expected one of {}, saw {}.'.format(expected_ctxs, ctx_type)) + ctx = mx.cpu(0) if ctx_type == 'cpu' else mx.gpu(0) + ctx_list.append({'ctx': ctx, 'pool_data': data, 'type_dict': {'pool_data': dtype}}) + # start with pool args present in all cases + pool_op_args = {'kernel': kernel, 'pool_type': pool_type, + 'pooling_convention' : pooling_convention, 'name' : 'pool'} + # add other args as needed + if global_pool: + pool_op_args['global_pool'] = True else: - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu', 'pool_cudnn'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='full', global_pool=False, - count_include_pad=count_include_pad) - - # pool_v1 has bugs when pad is not 0, do not test PoolingV1 here - pad = (2, 3) - stride = (2, 3) - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu', 'pool_cudnn'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='full', global_pool=False, p_value=p_value, - count_include_pad=count_include_pad) - - if pool_type == 'lp': - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu', 'pool_cudnn'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - global_pool=True, p_value=p_value) - else: - test_pooling_versions_helper(pool_op_list=['pool_v1_cpu', 'pool_v1_gpu', 'pool_cpu', 'pool_gpu', 'pool_cudnn'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - global_pool=True, count_include_pad=count_include_pad) - - def test_3d_pooling(pool_type, p_value=2, count_include_pad=True): - data = (2, 3, 20, 20, 20) - kernel = (4, 5, 3) - pad = (0, 0, 0) - stride = (1, 1, 1) - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu', 'pool_cudnn'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='valid', global_pool=False, p_value=p_value, - count_include_pad=count_include_pad) - - pad = (2, 3, 3) - stride = (2, 3, 1) - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu', 'pool_cudnn'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='valid', global_pool=False, p_value=p_value, - count_include_pad=count_include_pad) - - pad = (0, 0, 0) - stride = (1, 1, 1) - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu', 'pool_cudnn'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='full', global_pool=False, p_value=p_value, - count_include_pad=count_include_pad) - - pad = (2, 3, 3) - stride = (2, 3, 1) - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu', 'pool_cudnn'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - pooling_convention='full', global_pool=False, p_value=p_value, - count_include_pad=count_include_pad) - - test_pooling_versions_helper(pool_op_list=['pool_cpu', 'pool_gpu', 'pool_cudnn'], - data=data, kernel=kernel, pad=pad, stride=stride, pool_type=pool_type, - global_pool=True, p_value=p_value, count_include_pad=count_include_pad) - - test_1d_pooling('max') - test_1d_pooling('avg', count_include_pad=True) - test_1d_pooling('avg', count_include_pad=False) - test_1d_pooling('sum') - test_1d_pooling('lp', p_value=1) - test_1d_pooling('lp', p_value=2) - test_1d_pooling('lp', p_value=3) + # Add pad and stride param if needed, plus randomly when it matches the default + if not is_default_pad(pad) or random_choice(): + pool_op_args.update({'pad' : pad}) + if not is_default_stride(stride) or random_choice(): + pool_op_args.update({'stride' : stride}) + + expected_pool_ops = ['pool', 'pool_transposed', 'pool_v1'] + if pool_op == 'pool_v1': + sym = mx.sym.Pooling_v1(**pool_op_args) + else: + pool_op_args.update({'p_value' : p_value, 'count_include_pad' : count_include_pad}) + if ctx_type != 'cpu': + pool_op_args['cudnn_off'] = ctx_type == 'gpu' + if pool_op == 'pool': + # isolate pooling input from symbol input to test shared tensor optimizations + buffered_input = mx.sym.identity(name='pool') + sym = mx.sym.Pooling(buffered_input, **pool_op_args) + elif pool_op == 'pool_transposed': + ndim = len(data) + # NCW->NWC axes=(0,2,1) NCHW->NHWC axes=(0,2,3,1) NCDHW->NDHWC axes=(0,2,3,4,1); + axes = (0,) + tuple(range(2,ndim)) + (1,) + transposed = mx.sym.transpose(axes=axes, name='pool') + pooled = mx.sym.Pooling(data=transposed, layout=transposed_layout(ndim), + **pool_op_args) + # NWC->NCW axes=(0,2,1) NHWC->NCHW axes=(0,3,1,2) NDHWC->NCDHW axes=(0,4,1,2,3); + axes = (0, ndim-1) + tuple(range(1,ndim-1)) + sym = mx.sym.transpose(data=pooled, axes=axes, name='pool') + else: + raise RuntimeError('Expected one of {}, saw {}.'.format(expected_pool_ops, + pool_op)) + sym_list.append(sym) - test_2d_pooling('max') - test_2d_pooling('avg', count_include_pad=True) - test_2d_pooling('avg', count_include_pad=False) - test_2d_pooling('sum') - test_2d_pooling('lp', p_value=1) - test_2d_pooling('lp', p_value=2) - test_2d_pooling('lp', p_value=3) + check_consistency(sym_list, ctx_list, equal_nan=(not count_include_pad), tol=tol) - test_3d_pooling('max') - test_3d_pooling('avg', count_include_pad=True) - test_3d_pooling('avg', count_include_pad=False) - test_3d_pooling('sum') - test_3d_pooling('lp', p_value=1) - test_3d_pooling('lp', p_value=2) - test_3d_pooling('lp', p_value=3) + def test_pooling_dim(dim, pool_type, dtype, pool_op_list, p_value=2, count_include_pad=True, + tol=None): + if dim == '1D': + data = (3, 3, 10) + kernels = [(4,), (4,), (5,)] + pads = [(0,), (2,), (2,)] + strides = [(1,), (2,), (1,)] + elif dim == '2D_no_padding': + data = (3, 2, 20, 20) + kernels = [(3, 3), (4, 5)] + pads = [(0, 0), (0, 0)] + strides = [(1, 1), (2, 1)] + elif dim == '2D': + data = (2, 2, 20, 20) + kernels = [(3, 3), (3, 5), (4, 5), (4, 5)] + pads = [(0, 0), (1, 2), (0, 0), (2, 3)] + strides = [(1, 1), (1, 1), (2, 1), (1, 1)] + elif dim == '3D': + data = (2, 3, 20, 20, 20) + kernels = [(4, 5, 3), (4, 5, 3), (3, 5, 7)] + pads = [(0, 0, 0), (2, 3, 2), (1, 2, 3)] + strides = [(1, 1, 1), (2, 3, 1), (1, 1, 1)] + else: + raise RuntimeError('Unexpected pooling test class: {}.'.format(dim)) + + for kernel, pad, stride in zip(kernels, pads, strides): + for pooling_convention in ['valid', 'full']: + try: + test_pooling_versions_helper(pool_op_list=pool_op_list, + data=data, kernel=kernel, pad=pad, stride=stride, + pool_type=pool_type, pooling_convention=pooling_convention, + global_pool=False, p_value=p_value, + count_include_pad=count_include_pad, tol=tol, dtype=dtype) + except: + print('pool_op_list = {}'.format(pool_op_list)) + print('kernel={}, pad={}, stride={}'.format(kernel, pad, stride)) + print('pool_type={}, pooling_convention={}, global_pool=False'.format(pool_type, + pooling_convention)) + print('p_value={}, count_include_pad={}, dtype={}'.format(p_value, + count_include_pad, dtype)) + print('environ = \n{}'.format(os.environ)) + raise + + # Make sure kernel is ignored during global_pool by sometimes setting it to a crazy value + kernel = kernels[0] + if random_choice(): + kernel = (300,) * len(kernel) + + test_pooling_versions_helper(pool_op_list=pool_op_list, + data=data, kernel=kernel, pad=None, stride=None, + pool_type=pool_type, global_pool=True, p_value=p_value, + count_include_pad=count_include_pad, tol=tol, dtype=dtype) + + # The various implementations of the standard pooling operator + std_pool_op_list = ['pool_cpu', 'pool_transposed_cpu', + 'pool_gpu', 'pool_transposed_gpu', + 'pool_cudnn', 'pool_transposed_cudnn'] + # The implementations of the 'v1' pooling operator + v1_pool_op_list = ['pool_v1_cpu', 'pool_v1_gpu'] + # For those cases when all implementations should match- the combined implementation list. + combo_pool_op_list = std_pool_op_list + v1_pool_op_list + + for dtype in [np.float32, np.float64, np.float16]: + # Testing of the standard (not 'v1') pooling operator is universal across all + # data dimensions, implementations and layouts. + for dim in ['1D', '2D', '3D']: + test_pooling_dim(dim, 'max', dtype, std_pool_op_list) + test_pooling_dim(dim, 'avg', dtype, std_pool_op_list, count_include_pad=True) + test_pooling_dim(dim, 'avg', dtype, std_pool_op_list, count_include_pad=False) + test_pooling_dim(dim, 'sum', dtype, std_pool_op_list) + test_pooling_dim(dim, 'lp', dtype, std_pool_op_list, p_value=1) + test_pooling_dim(dim, 'lp', dtype, std_pool_op_list, p_value=2) + test_pooling_dim(dim, 'lp', dtype, std_pool_op_list, p_value=3) + + # Testing of the 'v1' pooling operator is over its restricted support domain of + # 2D data only and not with the 'lp' pooling type. The 'v1' cpu and gpu versions are + # always tested against each other, and sometimes against the standard operator versions. + # The slightly different 'v1' definition prevents this in the following cases: + # + # 1. In max pooling, when multiple input values are the maximum in the input window, + # the 'v1' implementation backprops the gradient to all maxima, whereas the standard + # pooling operator backprops the gradient to the lowest-indexed maximum only. + # 2. In max pooling, the 'v1' operator pads with 0's and this value can become the + # maximum output value in the case of an all-negative input. The standard pooling + # operator effectively considers the padding to be the largest negative value, so + # only input values should appear in the output. + # 3. In avg pooling, the 'v1' operator divides the sum by the same window size factor, + # even at the edges, and so does not support count_include_pad = False. + # 4. The float16 'v1' pooling operator performs forward sums and averages in + # float16, whereas the std operators perform those calculations in float32, so + # greater float16 tolerances are needed when comparing across implementations. + + # Double the float16 tol when comparing v1 and non-v1 implemenations, per note 4 above. + relaxed_tol = {np.dtype(np.float16): 2e-1, + np.dtype(np.float32): 1e-3, + np.dtype(np.float64): 1e-5, + np.dtype(np.uint8): 0, + np.dtype(np.int32): 0, + np.dtype(np.int64): 0} + + # Exclude std implementations due to points 1 and 2 above. + test_pooling_dim('2D', 'max', dtype, v1_pool_op_list) + # The standard and 'v1' implementations match for this case. + test_pooling_dim('2D', 'avg', dtype, combo_pool_op_list, count_include_pad=True, + tol=relaxed_tol) + # Exclude std implementations due to point 3 above. + test_pooling_dim('2D', 'avg', dtype, v1_pool_op_list, count_include_pad=False) + # The standard and 'v1' implementations match for this case. + test_pooling_dim('2D', 'sum', dtype, combo_pool_op_list, tol=relaxed_tol) + + # We can compare the standard and 'v1' max pooling implementations if we eliminate padding + # (see point 2 above) and use np.float64 data so that no two random input window values are + # likely to be the same (see point 1 above). + test_pooling_dim('2D_no_padding', 'max', np.float64, combo_pool_op_list) @with_seed() diff --git a/tests/python/unittest/test_gluon.py b/tests/python/unittest/test_gluon.py index abe6b136fe0c..34380dc00314 100644 --- a/tests/python/unittest/test_gluon.py +++ b/tests/python/unittest/test_gluon.py @@ -506,50 +506,75 @@ def test_deconv(): @with_seed() def test_pool(): - layers1d = [ - nn.MaxPool1D(), - nn.MaxPool1D(3), - nn.MaxPool1D(3, 2), - nn.AvgPool1D(), - nn.AvgPool1D(count_include_pad=False), - nn.GlobalAvgPool1D(), - ] - for layer in layers1d: - check_layer_forward(layer, (1, 2, 10)) - - - layers2d = [ - nn.MaxPool2D(), - nn.MaxPool2D((3, 3)), - nn.MaxPool2D(3, 2), - nn.AvgPool2D(), - nn.AvgPool2D(count_include_pad=False), - nn.GlobalAvgPool2D(), - ] - for layer in layers2d: - check_layer_forward(layer, (1, 2, 10, 10)) - - layers3d = [ - nn.MaxPool3D(), - nn.MaxPool3D((3, 3, 3)), - nn.MaxPool3D(3, 2), - nn.AvgPool3D(), - nn.AvgPool3D(count_include_pad=False), - nn.GlobalAvgPool3D(), - ] - for layer in layers3d: - check_layer_forward(layer, (1, 2, 10, 10, 10)) + # transpose shape to bring feature dimension 'c' from 2nd position to last + def transpose(shape): + return (shape[0],) + shape[2:] + (shape[1],) + + for layout in ['NCW', 'NWC']: + shape1d = (1, 2, 10) + if layout == 'NWC': + shape1d = transpose(shape1d) + layers1d = [ + nn.MaxPool1D(layout=layout), + nn.MaxPool1D(3, layout=layout), + nn.MaxPool1D(3, 2, layout=layout), + nn.AvgPool1D(layout=layout), + nn.AvgPool1D(count_include_pad=False, layout=layout), + nn.GlobalAvgPool1D(layout=layout), + ] + for layer in layers1d: + check_layer_forward(layer, shape1d) + + + for layout in ['NCHW', 'NHWC']: + shape2d = (1, 2, 10, 10) + if layout == 'NHWC': + shape2d = transpose(shape2d) + layers2d = [ + nn.MaxPool2D(layout=layout), + nn.MaxPool2D((3, 3), layout=layout), + nn.MaxPool2D(3, 2, layout=layout), + nn.AvgPool2D(layout=layout), + nn.AvgPool2D(count_include_pad=False, layout=layout), + nn.GlobalAvgPool2D(layout=layout), + ] + for layer in layers2d: + check_layer_forward(layer, shape2d) + + for layout in ['NCDHW', 'NDHWC']: + shape3d = (1, 2, 10, 10, 10) + if layout == 'NDHWC': + shape3d = transpose(shape3d) + layers3d = [ + nn.MaxPool3D(layout=layout), + nn.MaxPool3D((3, 3, 3), layout=layout), + nn.MaxPool3D(3, 2, layout=layout), + nn.AvgPool3D(layout=layout), + nn.AvgPool3D(count_include_pad=False, layout=layout), + nn.GlobalAvgPool3D(layout=layout), + ] + for layer in layers3d: + check_layer_forward(layer, shape3d) # test ceil_mode - x = mx.nd.zeros((2, 2, 10, 10)) + for layout in ['NCHW', 'NHWC']: + xshape = (2, 2, 10, 10) + noceil_out_shape = (2, 2, 3, 3) + ceil_out_shape = (2, 2, 4, 4) + if layout == 'NHWC': + xshape = transpose(xshape) + noceil_out_shape = transpose(noceil_out_shape) + ceil_out_shape = transpose(ceil_out_shape) - layer = nn.MaxPool2D(3, ceil_mode=False) - layer.collect_params().initialize() - assert (layer(x).shape==(2, 2, 3, 3)) + x = mx.nd.zeros(xshape) - layer = nn.MaxPool2D(3, ceil_mode=True) - layer.collect_params().initialize() - assert (layer(x).shape==(2, 2, 4, 4)) + layer = nn.MaxPool2D(3, ceil_mode=False, layout=layout) + layer.collect_params().initialize() + assert (layer(x).shape==noceil_out_shape) + + layer = nn.MaxPool2D(3, ceil_mode=True, layout=layout) + layer.collect_params().initialize() + assert (layer(x).shape==ceil_out_shape) @with_seed() @@ -2091,31 +2116,41 @@ def hybrid_forward(self, F, x): @with_seed() def test_slice_pooling2d(): - max_pooling = nn.MaxPool2D(strides=(2, 3), padding=(1, 1)) - avg_pooling = nn.AvgPool2D(strides=(2, 2), padding=(1, 1)) - global_maxpooling = nn.GlobalMaxPool2D() - global_avgpooling = nn.GlobalAvgPool2D() - pooling_layers = [max_pooling, avg_pooling, global_maxpooling, global_avgpooling] - class Net(gluon.HybridBlock): - def __init__(self, - slice, - pooling_layer, - **kwargs): - super(Net, self).__init__(**kwargs) - with self.name_scope(): - self.slice = slice - self.pool0 = pooling_layer - - def hybrid_forward(self, F, x): - x_slice = x.slice(begin=self.slice[0], end=self.slice[1]) - out = self.pool0(x_slice) - return out - - x = mx.nd.random.uniform(shape=(16, 128, 256, 256)) - slice = [(0, 0, 0, 0), (4, 16, 32, 64)] - for i in range(len(pooling_layers)): - net = Net(slice, pooling_layers[i]) - check_layer_forward_withinput(net, x) + # transpose shape to bring feature dimension 'c' from 2nd position to last + def transpose(shape): + return (shape[0],) + shape[2:] + (shape[1],) + + for layout in ['NCHW', 'NHWC']: + max_pooling = nn.MaxPool2D(strides=(2, 3), padding=(1, 1), layout=layout) + avg_pooling = nn.AvgPool2D(strides=(2, 2), padding=(1, 1), layout=layout) + global_maxpooling = nn.GlobalMaxPool2D(layout=layout) + global_avgpooling = nn.GlobalAvgPool2D(layout=layout) + pooling_layers = [max_pooling, avg_pooling, global_maxpooling, global_avgpooling] + class Net(gluon.HybridBlock): + def __init__(self, + slice, + pooling_layer, + **kwargs): + super(Net, self).__init__(**kwargs) + with self.name_scope(): + self.slice = slice + self.pool0 = pooling_layer + + def hybrid_forward(self, F, x): + x_slice = x.slice(begin=self.slice[0], end=self.slice[1]) + out = self.pool0(x_slice) + return out + + xshape = (16, 128, 256, 256) + slice_shape = (4, 16, 32, 64) + if layout == 'NHWC': + xshape = transpose(xshape) + slice_shape = transpose(slice_shape) + x = mx.nd.random.uniform(shape=xshape) + slice = [(0, 0, 0, 0), slice_shape] + for i in range(len(pooling_layers)): + net = Net(slice, pooling_layers[i]) + check_layer_forward_withinput(net, x) @with_seed() @unittest.skip('skippping temporarily, tracked by https://github.com/apache/incubator-mxnet/issues/11164') From bada8a1961f0da7f01cd9e61d03f280c48083f1b Mon Sep 17 00:00:00 2001 From: Xinyu Chen Date: Tue, 19 Feb 2019 02:57:22 +0800 Subject: [PATCH 41/75] add quantization example to readme (#14186) --- example/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/example/README.md b/example/README.md index cd49cf9ff9af..18022beb7598 100644 --- a/example/README.md +++ b/example/README.md @@ -128,6 +128,7 @@ If your tutorial depends on specific packages, simply add them to this provision * [Numpy Operator Customization](numpy-ops) - Examplea on quick customize new ops with Numpy * [Profiling](profiler) - generate profiling results in json files * [Python How To](python-howto) - a variety of Python examples +* [Quantization and Calibration Examples](quantization) - examples of quantizing a FP32 model to INT8 and performing low-precision inference with Intel MKL-DNN on CPU or cuDNN on GPU * [R-CNN](rcnn) - R-CNN with distributed implementation and data parallelization * [Recommender Systems](recommenders) - examples of how to build various kinds of recommender systems * [Reinforcement Learning](reinforcement-learning) - a variety of reinforcement learning examples From 3a6fe22b660797f0b6f7d0530c604024b6f84bae Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 18 Feb 2019 20:29:47 +0100 Subject: [PATCH 42/75] =?UTF-8?q?Refine=20runtime=20feature=20discovery=20?= =?UTF-8?q?python=20API=20and=20add=20documentation=20to=20=E2=80=A6=20(#1?= =?UTF-8?q?4130)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Refine runtime feature discovery python API and add documentation to Python API docs * Fix lint * Provide is_enabled method to check if feature is present from string * Refine docs, add is_enabled * Fix encoding * Fix doc * Address CR suggestions * remove index as per CR suggestion * Fix lint * runtime * Fix doc * Add license --- docs/api/python/index.md | 11 +++++ docs/api/python/libinfo/libinfo.md | 69 +++++++++++++++++++++++++++ include/mxnet/c_api.h | 1 - python/mxnet/runtime.py | 59 +++++++++++++++++++---- src/libinfo.cc | 1 - tests/python/unittest/test_runtime.py | 22 ++++++--- 6 files changed, 145 insertions(+), 18 deletions(-) create mode 100644 docs/api/python/libinfo/libinfo.md diff --git a/docs/api/python/index.md b/docs/api/python/index.md index 6130078ee796..2e23b162ee77 100644 --- a/docs/api/python/index.md +++ b/docs/api/python/index.md @@ -174,6 +174,17 @@ Code examples are placed throughout the API documentation and these can be run a rtc/rtc.md ``` +## Run-Time Feature detection / Library Info + +```eval_rst +.. toctree:: + :maxdepth: 1 + + libinfo/libinfo.md +``` + + + ## Symbol API ```eval_rst diff --git a/docs/api/python/libinfo/libinfo.md b/docs/api/python/libinfo/libinfo.md new file mode 100644 index 000000000000..531e1ced3c99 --- /dev/null +++ b/docs/api/python/libinfo/libinfo.md @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + +# Run-Time Feature detection / Library info + +```eval_rst +.. currentmodule:: mxnet.runtime +``` + +## Overview + +The libinfo functionality allows to check for compile-time features supported by the library. + +### Example usage + +``` +In [1]: import mxnet as mx + ...: import mxnet.runtime + ...: fs = mx.runtime.Features() + +In [2]: fs +Out[2]: [✖ CUDA, ✖ CUDNN, ✖ NCCL, ✖ CUDA_RTC, ✖ TENSORRT, ✔ CPU_SSE, ✔ CPU_SSE2, ✔ CPU_SSE3, ✔ CPU_SSE4_1, ✔ CPU_SSE4_2, ✖ CPU_SSE4A, ✔ CPU_AVX, ✖ CPU_AVX2, ✖ OPENMP, ✖ SSE, ✔ F16C, ✖ JEMALLOC, ✔ BLAS_OPEN, ✖ BLAS_ATLAS, ✖ BLAS_MKL, ✖ BLAS_APPLE, ✔ LAPACK, ✖ MKLDNN, ✔ OPENCV, ✖ CAFFE, ✖ PROFILER, ✖ DIST_KVSTORE, ✖ CXX14, ✔ SIGNAL_HANDLER, ✔ DEBUG] + +In [3]: fs['CUDA'].enabled +Out[3]: False + +In [4]: fs.is_enabled('CPU_SSE') +Out[4]: True + +In [5]: fs.is_enabled('CUDA') +Out[5]: False + +In [6]: +``` + + +```eval_rst +.. autosummary:: + :nosignatures: + + Features + Feature + feature_list +``` + +## API Reference + + + +```eval_rst +.. automodule:: mxnet.runtime + :members: +``` + + diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index e5e57c10faaa..13ee903407b3 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -141,7 +141,6 @@ struct MXCallbackList { struct LibFeature { const char* name; - uint32_t index; bool enabled; }; diff --git a/python/mxnet/runtime.py b/python/mxnet/runtime.py index afb393281420..7ef5e1943072 100644 --- a/python/mxnet/runtime.py +++ b/python/mxnet/runtime.py @@ -1,3 +1,5 @@ +# coding: utf-8 + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,34 +17,71 @@ # specific language governing permissions and limitations # under the License. -# coding: utf-8 # pylint: disable=not-an-iterable """runtime querying of compile time features in the native library""" import ctypes +import collections from .base import _LIB, check_call -class LibFeature(ctypes.Structure): +class Feature(ctypes.Structure): """ Compile time feature description """ _fields_ = [ - ("name", ctypes.c_char_p), - ("index", ctypes.c_uint32), + ("_name", ctypes.c_char_p), ("enabled", ctypes.c_bool) ] -def libinfo_features(): + @property + def name(self): + return self._name.decode() + + def __repr__(self): + if self.enabled: + return "✔ {}".format(self.name) + else: + return "✖ {}".format(self.name) + +def feature_list(): """ Check the library for compile-time features. The list of features are maintained in libinfo.h and libinfo.cc Returns ------- - A list of class LibFeature indicating which features are available and enabled + :return: list of class LibFeature indicating which features are available and enabled """ - lib_features = ctypes.POINTER(LibFeature)() + lib_features_c_array = ctypes.POINTER(Feature)() lib_features_size = ctypes.c_size_t() - check_call(_LIB.MXLibInfoFeatures(ctypes.byref(lib_features), ctypes.byref(lib_features_size))) - feature_list = [lib_features[i] for i in range(lib_features_size.value)] - return feature_list + check_call(_LIB.MXLibInfoFeatures(ctypes.byref(lib_features_c_array), ctypes.byref(lib_features_size))) + features = [lib_features_c_array[i] for i in range(lib_features_size.value)] + return features + +class Features(collections.OrderedDict): + """ + OrderedDict of name to Feature + """ + def __init__(self): + super(Features, self).__init__([(f.name, f) for f in feature_list()]) + + def __repr__(self): + return str(list(self.values())) + + def is_enabled(self, feature_name): + """ + Check for a particular feature by name + + Parameters + ---------- + :param x: str The name of a valid feature as string for example 'CUDA' + + Returns + ------- + :return: bool True if it's enabled, False if it's disabled, RuntimeError if the feature is not known + """ + feature_name = feature_name.upper() + if feature_name not in self: + raise RuntimeError("Feature '{}' is unknown, known features are: {}".format( + feature_name, list(self.keys()))) + return self[feature_name].enabled diff --git a/src/libinfo.cc b/src/libinfo.cc index 44a834c85b16..2af61eac9eca 100644 --- a/src/libinfo.cc +++ b/src/libinfo.cc @@ -114,7 +114,6 @@ LibInfo::LibInfo() { for (size_t i = 0; i < MAX_FEATURES; ++i) { m_lib_features[i].name = EnumNames::names[i].c_str(); m_lib_features[i].enabled = is_enabled(i); - m_lib_features[i].index = i; } } diff --git a/tests/python/unittest/test_runtime.py b/tests/python/unittest/test_runtime.py index 433301819252..5b06d0c3c36c 100644 --- a/tests/python/unittest/test_runtime.py +++ b/tests/python/unittest/test_runtime.py @@ -21,14 +21,24 @@ from mxnet.base import MXNetError from nose.tools import * -def test_libinfo_features(): - features = libinfo_features() - print("Lib features: ") +def test_features(): + features = Features() + print(features) + ok_('CUDA' in features) + ok_(len(features) >= 30) + +def test_is_enabled(): + features = Features() for f in features: - print(f.name, f.enabled, f.index) - ok_(type(features) is list) - ok_(len(features) > 0) + if features[f].enabled: + ok_(features.is_enabled(f)) + else: + ok_(not features.is_enabled(f)) +@raises(RuntimeError) +def test_is_enabled_not_existing(): + features = Features() + features.is_enabled('this girl is on fire') if __name__ == "__main__": From af9e3b3ae9f55b51dd2557a8b664622571ceda48 Mon Sep 17 00:00:00 2001 From: Aaron Markham Date: Tue, 19 Feb 2019 07:08:39 -0800 Subject: [PATCH 43/75] update raspberry pi install instructions (#14172) --- docs/install/index.md | 50 ++++++++++++++++++++++++++++++---- docs/install/raspbian_setup.md | 4 +-- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/docs/install/index.md b/docs/install/index.md index 76f22e744075..2b7e0457e2b4 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -1188,7 +1188,7 @@ MXNet should work on any cloud provider's CPU-only instances. Follow the Python
-MXNet supports the Debian based Raspbian ARM based operating system so you can run MXNet on Raspberry Pi Devices. +MXNet supports the Debian based Raspbian ARM based operating system so you can run MXNet on Raspberry Pi 3B devices. These instructions will walk through how to build MXNet for the Raspberry Pi and install the Python bindings for the library. @@ -1196,6 +1196,9 @@ You can do a dockerized cross compilation build on your local machine or a nativ The complete MXNet library and its requirements can take almost 200MB of RAM, and loading large models with the library can take over 1GB of RAM. Because of this, we recommend running MXNet on the Raspberry Pi 3 or an equivalent device that has more than 1 GB of RAM and a Secure Digital (SD) card that has at least 4 GB of free memory. +## Quick installation +You can use this [pre-built Python wheel](wget https://mxnet-public.s3.amazonaws.com/install/raspbian/mxnet-1.5.0-py2.py3-none-any.whl) on a Raspberry Pi 3B with Stretch. You will likely need to install several dependencies to get MXNet to work. Refer to the following **Build** section for details. + **Cross compilation build (Experimental)** ## Docker installation @@ -1222,11 +1225,48 @@ ci/build.py -p armv7 ## Install -Create a virtualenv and install the package we created previously. +Your Pi will need several dependencies. + +Install MXNet dependencies with the following: +``` +sudo apt-get update +sudo apt-get install -y \ + apt-transport-https \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + git \ + libatlas-base-dev \ + libcurl4-openssl-dev \ + libjemalloc-dev \ + liblapack-dev \ + libopenblas-dev \ + libopencv-dev \ + libzmq3-dev \ + ninja-build \ + python-dev \ + software-properties-common \ + sudo \ + unzip \ + virtualenv \ + wget +``` +Install virtualenv with: +``` +sudo pip install virtualenv +``` +Create a Python 2.7 environment for MXNet with: +``` +virtualenv -p `which python` mxnet_py27 +``` +You may use Python 3, however the [wine bottle detection example](https://mxnet.incubator.apache.org/versions/master/tutorials/embedded/wine_detector.html) for the Pi with camera requires Python 2.7. + +Create a virtualenv and install the wheel we created previously, or the wheel that you downloaded. ``` -virtualenv -p `which python3` mxnet_py3 -source mxnet_py3/bin/activate +virtualenv -p `which python3` mxnet_py27 +source mxnet_py27/bin/activate pip install mxnet-x.x.x-py2.py3-none-any.whl ``` @@ -1257,7 +1297,7 @@ Install these dependencies using the following commands in any directory: ``` sudo apt-get update - sudo apt-get -y install git cmake ninja-build build-essential g++-4.9 c++-4.9 liblapack* libblas* libopencv* libopenblas* python3-dev virtualenv + sudo apt-get -y install git cmake ninja-build build-essential g++-4.9 c++-4.9 liblapack* libblas* libopencv* libopenblas* python3-dev python-dev virtualenv ``` Clone the MXNet source code repository using the following `git` command in your home directory: diff --git a/docs/install/raspbian_setup.md b/docs/install/raspbian_setup.md index a432d4815662..896d4721370b 100644 --- a/docs/install/raspbian_setup.md +++ b/docs/install/raspbian_setup.md @@ -17,9 +17,9 @@ - +

- + This content is moved to a new MXNet install page. Redirecting...

From 49c311cb2377093bcc13abdb3845401b789892ee Mon Sep 17 00:00:00 2001 From: Jake Lee Date: Wed, 20 Feb 2019 04:35:51 +0800 Subject: [PATCH 44/75] Add tutorial on how to use build from source jar (#14197) * add tutorial * fix the typo --- scala-package/README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/scala-package/README.md b/scala-package/README.md index 8322ab2a237f..c7d0cecf15ac 100644 --- a/scala-package/README.md +++ b/scala-package/README.md @@ -179,6 +179,37 @@ mvn deploy -Pstaging Examples & Usage ------- +Assuming you use `mvn install`, you can find the `mxnet-full_scala_version-INTERNAL.jar` e.g. `mxnet-full_2.11-INTERNAL.jar` under the path `incubator-mxnet/scala-package/assembly/target`. + +Adding the following configuration in `pom.xml` +```HTML + + org.apache.mxnet + mxnet-full_2.11-INTERNAL + 1.5.0 + system + path_to_jar/mxnet-full_2.11-INTERNAL.jar + +``` +If you have following error message +``` +Error: A JNI error has occurred, please check your installation and try again +Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/mxnet/NDArray + at java.lang.Class.getDeclaredMethods0(Native Method) + at java.lang.Class.privateGetDeclaredMethods(Class.java:2701) + at java.lang.Class.privateGetMethodRecursive(Class.java:3048) + at java.lang.Class.getMethod0(Class.java:3018) + at java.lang.Class.getMethod(Class.java:1784) + at sun.launcher.LauncherHelper.validateMainClass(LauncherHelper.java:544) + at sun.launcher.LauncherHelper.checkAndLoadMain(LauncherHelper.java:526) +Caused by: java.lang.ClassNotFoundException: org.apache.mxnet.NDArray + at java.net.URLClassLoader.findClass(URLClassLoader.java:381) + at java.lang.ClassLoader.loadClass(ClassLoader.java:424) + at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331) + at java.lang.ClassLoader.loadClass(ClassLoader.java:357) +``` +Please make sure your $CLASSPATH is able to find `mxnet-full_scala_version-INTERNAL.jar`. + - To set up the Scala Project using IntelliJ IDE on macOS follow the instructions [here](https://mxnet.incubator.apache.org/tutorials/scala/mxnet_scala_on_intellij.html). - Several examples on using the Scala APIs are provided in the [Scala Examples Folder](https://github.com/apache/incubator-mxnet/tree/master/scala-package/examples/) From c45e53462da69975d180fe672a77f73c8b641460 Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Tue, 19 Feb 2019 12:36:47 -0800 Subject: [PATCH 45/75] Scala interpreter instructions (#14169) --- docs/install/scala_setup.md | 28 ++++++++++++++++++++++++++++ docs/tutorials/scala/char_lstm.md | 6 +----- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/install/scala_setup.md b/docs/install/scala_setup.md index bc069a14e6b3..15a2def1ef38 100644 --- a/docs/install/scala_setup.md +++ b/docs/install/scala_setup.md @@ -20,6 +20,7 @@ The following instructions are provided for macOS and Ubuntu. Windows is not yet available. **Note:** If you use IntelliJ or a similar IDE, you may want to follow the [MXNet-Scala on IntelliJ tutorial](../tutorials/scala/mxnet_scala_on_intellij.html) instead of these instructions. +**Note:** Currently, we only support scala 2.11
@@ -114,6 +115,33 @@ mvn install
+## Interpreter + +To run the scala interpreter, first download and install scala 2.11.x (run `scala -version` to make sure you have the right version installed.** + +### Installing the Interpreter + +**Ubuntu*** + +``` +sudo apt-get install scala +``` + +**macOS*** + +``` +brew install scala@2.11 +``` + +Then, add scala to your path by following the instructions output by homebrew. + +### Running the Interpreter + +To run the interpreter, download the appropriate mxnet jar from [the maven repository](https://search.maven.org/search?q=g:org.apache.mxnet) or build from source following the instructions above. + +Then, run `scala -cp {path/to/mxnet-full_2.11-os-version.jar}` to start it. +If you receive a "NumberFormatException" when running the interpreter, run `export TERM=xterm-color` before starting the interpreter. + ## Documentation Scaladocs are generated as part of the docs build pipeline. You can find them published in the [Scala API](http://mxnet.incubator.apache.org/api/scala/index.html) section of the website or by going to the [scaladocs output](https://mxnet.incubator.apache.org/api/scala/docs/index.html#org.apache.mxnet.package) directly. diff --git a/docs/tutorials/scala/char_lstm.md b/docs/tutorials/scala/char_lstm.md index 972661bc81ef..aca08dc79920 100644 --- a/docs/tutorials/scala/char_lstm.md +++ b/docs/tutorials/scala/char_lstm.md @@ -71,11 +71,7 @@ In this tutorial, you will accomplish the following: ## Prerequisites -To complete this tutorial, you need: - -- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/install/index.html) -- [Scala 2.11.8](https://www.scala-lang.org/download/2.11.8.html) -- [Maven 3](https://maven.apache.org/install.html) +To complete this tutorial, setup and run the scala interpreter by following the [instructions](https://mxnet.incubator.apache.org/install/scala_setup.html#interpreter). ## Download the Data From b396a63bf99652bf3ee8c6c9135b73113127fcce Mon Sep 17 00:00:00 2001 From: Nathalie Rauschmayr Date: Tue, 19 Feb 2019 13:26:24 -0800 Subject: [PATCH 46/75] Fix READMEs for examples (#14179) * fixing examples * fixing examples and adding READMEs fixing examples and adding READMEs fixing examples and adding READMEs fixing examples and adding READMEs add missing files fixed some bugs in readmes fixed some bugs in readmes fixed some spelling mistakes update update * removing images and fixing spelling mistakes * removing images and fixing spelling mistakes * updating links * Adding license header * empty commit * Add missing license headers * Adding changes requested by ChaiBapchya --- example/captcha/mxnet_captcha.R | 4 +- example/gan/CGAN_mnist_R/README.md | 168 +++++++++++++++++++ example/gluon/actor_critic/README.md | 61 +++++++ example/gluon/audio/README.md | 115 +++++++++++++ example/gluon/house_prices/README.md | 29 ++++ example/gluon/lstm_crf/README.md | 36 ++++ example/gluon/mnist/README.md | 55 ++++++ example/gluon/super_resolution/README.md | 45 +++++ example/gluon/{ => super_resolution}/data.py | 0 example/model-parallel/README.md | 20 +++ example/sparse/README.md | 24 +++ 11 files changed, 555 insertions(+), 2 deletions(-) create mode 100644 example/gan/CGAN_mnist_R/README.md create mode 100644 example/gluon/actor_critic/README.md create mode 100644 example/gluon/audio/README.md create mode 100644 example/gluon/house_prices/README.md create mode 100644 example/gluon/lstm_crf/README.md create mode 100644 example/gluon/mnist/README.md create mode 100644 example/gluon/super_resolution/README.md rename example/gluon/{ => super_resolution}/data.py (100%) create mode 100644 example/model-parallel/README.md create mode 100644 example/sparse/README.md diff --git a/example/captcha/mxnet_captcha.R b/example/captcha/mxnet_captcha.R index 8988d25104d5..43e819f8c264 100644 --- a/example/captcha/mxnet_captcha.R +++ b/example/captcha/mxnet_captcha.R @@ -39,9 +39,9 @@ label <- mx.symbol.Reshape(data = label, target_shape = c(0)) captcha_net <- mx.symbol.SoftmaxOutput(data = fc2, label = label, name = "softmax") mx.metric.acc2 <- mx.metric.custom("accuracy", function(label, pred) { - ypred <- max.col(t(pred)) - 1 + ypred <- max.col(t(data.matrix(pred))) - 1 ypred <- matrix(ypred, nrow = nrow(label), ncol = ncol(label), byrow = TRUE) - return(sum(colSums(label == ypred) == 4) / ncol(label)) + return(sum(colSums(data.matrix(label) == ypred) == 4) / ncol(label)) }) data.shape <- c(80, 30, 3) diff --git a/example/gan/CGAN_mnist_R/README.md b/example/gan/CGAN_mnist_R/README.md new file mode 100644 index 000000000000..bf0bb08b1147 --- /dev/null +++ b/example/gan/CGAN_mnist_R/README.md @@ -0,0 +1,168 @@ + + + + + + + + + + + + + + + + + +# Conditional Generative Adversarial Network with MXNet R package + +This tutorial shows how to build and train a Conditional Generative Adversarial Network (CGAN) on MNIST images. + +## How GAN works +A Generative Adversarial Model simultaneously trains two models: a generator that learns to output fake samples from an unknown distribution and a discriminator that learns to distinguish fake from real samples. + +The CGAN is a conditional variation of the GAN where the generator is instructed to generate a real sample having specific characteristics rather than a generic sample from full distribution. Such condition could be the label associated with an image like in this tutorial or a more detailed tag as shown in the example below: + +![Image credit: (Scott Reed)[https://github.com/reedscot/icml2016]](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gan/CGAN_mnist_R/dcgan_network.jpg) + +## Initial setup + +The following packages are needed to run the tutorial: + +``` +require("imager") +require("dplyr") +require("readr") +require("mxnet") +``` + +The full demo is comprised of the two following scripts: + +```CGAN_mnist_setup.R```: prepare data and define the model structure +```CGAN_train.R```: execute the training + +## Data preperation + +The MNIST dataset is available [here](https://www.kaggle.com/c/digit-recognizer/data)). Once train.csv is downloaded into the data/ folder, we can import into R. + +```train <- read_csv('data/train.csv') +train <- data.matrix(train) + +train_data <- train[,-1] +train_data <- t(train_data/255*2-1) +train_label <- as.integer(train[,1]) + +dim(train_data) <- c(28, 28, 1, ncol(train_data)) +``` +Custom iterators are defined in ```iterators.R``` and imported by ```CGAN_mnist_setup.R``` + +## Generator +The generator is a network that creates novel samples (MNIST images) from 2 inputs: + +- Noise vector +- Labels defining the object condition (which digit to produce) + +The noise vector provides the building blocks to the Generator model, which will learns how to structure that noise into a sample. The mx.symbol.Deconvolution operator is used to upsample the initial input from a 1x1 shape up to a 28x28 image. + +The information on the label for which to generate a fake sample is provided by a one-hot encoding of the label indices that is appended to the random noise. For MNIST, the 0-9 indices are therefore converted into a binary vector of length 10. More complex applications would require embeddings rather than simple one-hot to encode the condition. + +## Discriminator +The discriminator attempts to distinguish between fake samples produced by the generator and real ones sampled from MNIST training data. + +In a conditional GAN, the labels associated with the samples are also provided to the Discriminator. In this demo, this information is again provided as a hot-hot encoding of the label that is broadcast to match the image dimensions (10 -> 28x28x10). + +## Training logic +The training process of the discriminator is most obvious: the loss is simple a binary TRUE/FALSE response and that loss is propagated back into the CNN network. It can therefore be understood as a simple binary classification problem. + +```### Train loop on fake +mx.exec.update.arg.arrays(exec_D, arg.arrays = + list(data=D_data_fake, digit=D_digit_fake, label=mx.nd.array(rep(0, batch_size))), + match.name=TRUE) +mx.exec.forward(exec_D, is.train=T) +mx.exec.backward(exec_D) +update_args_D<- updater_D(weight = exec_D$ref.arg.arrays, grad = exec_D$ref.grad.arrays) +mx.exec.update.arg.arrays(exec_D, update_args_D, skip.null=TRUE) + +### Train loop on real +mx.exec.update.arg.arrays(exec_D, arg.arrays = + list(data=D_data_real, digit=D_digit_real, label=mx.nd.array(rep(1, batch_size))), + match.name=TRUE) +mx.exec.forward(exec_D, is.train=T) +mx.exec.backward(exec_D) +update_args_D<- updater_D(weight = exec_D$ref.arg.arrays, grad = exec_D$ref.grad.arrays) +mx.exec.update.arg.arrays(exec_D, update_args_D, skip.null=TRUE) +``` + +The generator loss comes from the backpropagation of the the discriminator loss into its generated output. By faking the generator labels to be real samples into the discriminator, the discriminator back-propagated loss provides the generator with the information on how to best adapt its parameters to trick the discriminator into believing the fake samples are real. + +This requires to backpropagate the gradients up to the input data of the discriminator (whereas this input gradient is typically ignored in vanilla feedforward network). + +```### Update Generator weights - use a seperate executor for writing data gradients +exec_D_back <- mxnet:::mx.symbol.bind(symbol = D_sym, + arg.arrays = exec_D$arg.arrays, + aux.arrays = exec_D$aux.arrays, grad.reqs = rep("write", length(exec_D$arg.arrays)), + ctx = devices) + +mx.exec.update.arg.arrays(exec_D_back, arg.arrays = + list(data=D_data_fake, digit=D_digit_fake, label=mx.nd.array(rep(1, batch_size))), + match.name=TRUE) +mx.exec.forward(exec_D_back, is.train=T) +mx.exec.backward(exec_D_back) +D_grads <- exec_D_back$ref.grad.arrays$data +mx.exec.backward(exec_G, out_grads=D_grads) + +update_args_G <- updater_G(weight = exec_G$ref.arg.arrays, grad = exec_G$ref.grad.arrays) +mx.exec.update.arg.arrays(exec_G, update_args_G, skip.null=TRUE) +``` + +The above training steps are executed in the ```CGAN_train.R``` script. + +## Monitor the training + +During training, the [imager](http://dahtah.github.io/imager/) package facilitates the visual quality assessment of the fake samples. + +```if (iteration==1 | iteration %% 100==0){ + par(mfrow=c(3,3), mar=c(0.1,0.1,0.1,0.1)) + for (i in 1:9) { + img <- as.array(exec_G$ref.outputs$G_sym_output)[,,,i] + plot(as.cimg(img), axes=F) + } +} +``` +Below are samples obtained at different stage of the training. + +Starting from noise: + +![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gan/CGAN_mnist_R/CGAN_1.png) + +Slowly getting it - iteration 200: + +![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gan/CGAN_mnist_R/CGAN_200.png) + +Generate specified digit images on demand - iteration 2400: + +![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gan/CGAN_mnist_R/CGAN_2400.png) + +## Inference + +Once the model is trained, synthetic images of the desired digit can be produced by feeding the generator with fixed labels rather than the randomly generated ones used during the training. + +Here we will generate fake ```9```: + +```digit <- mx.nd.array(rep(9, times=batch_size)) +data <- mx.nd.one.hot(indices = digit, depth = 10) +data <- mx.nd.reshape(data = data, shape = c(1,1,-1, batch_size)) + +exec_G <- mx.simple.bind(symbol = G_sym, data=data_shape_G, ctx = devices, grad.req = "null") +mx.exec.update.arg.arrays(exec_G, G_arg_params, match.name=TRUE) +mx.exec.update.arg.arrays(exec_G, list(data=data), match.name=TRUE) +mx.exec.update.aux.arrays(exec_G, G_aux_params, match.name=TRUE) + +mx.exec.forward(exec_G, is.train=F) +``` +![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gan/CGAN_mnist_R/CGAN_infer_9.png) + +Further details of the CGAN methodology can be found in the paper [Generative Adversarial Text to Image Synthesis](https://arxiv.org/abs/1605.05396). + + diff --git a/example/gluon/actor_critic/README.md b/example/gluon/actor_critic/README.md new file mode 100644 index 000000000000..7f3a6a73e972 --- /dev/null +++ b/example/gluon/actor_critic/README.md @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + +# Actor Critic Model + +This example shows an actor critic model that consists of a critic that measures how good an action taken is and an actor that controls the agent's behavior. +In our example actor and critic use the same model: + +``` +class Policy(gluon.Block): + def __init__(self, **kwargs): + super(Policy, self).__init__(**kwargs) + with self.name_scope(): + self.dense = nn.Dense(16, in_units=4, activation='relu') + self.action_pred = nn.Dense(2, in_units=16) + self.value_pred = nn.Dense(1, in_units=16) + + def forward(self, x): + x = self.dense(x) + probs = self.action_pred(x) + values = self.value_pred(x) + return F.softmax(probs), values +``` +The example uses [Gym](https://gym.openai.com/docs/), which is a toolkit for developing and comparing reinforcement learning algorithms. The model is running an instance of [CartPole-v0](https://gym.openai.com/envs/CartPole-v0/) that simulates a pole that is attached by an un-actuated joint to a cart, which moves along a frictionless track. The goal is to prevent it from falling over. + + +The example provides the following commandline options: +``` +MXNet actor-critic example + +optional arguments: + -h, --help show this help message and exit + --gamma G discount factor (default: 0.99) + --seed N random seed (default: 1) + --render render the environment + --log-interval N interval between training status logs (default: 10) + +``` + +To run the model execute, type +``` +python actor_critic.py --render +``` + +You will get an output like the following: +![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gluon/actor_critic/actor_critic.gif) + diff --git a/example/gluon/audio/README.md b/example/gluon/audio/README.md new file mode 100644 index 000000000000..cb2b53eb3b83 --- /dev/null +++ b/example/gluon/audio/README.md @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + +# Urban Sounds Classification in MXNet Gluon + +This example provides an end-to-end pipeline for a common datahack competition - [Urban Sounds Classification Example](https://datahack.analyticsvidhya.com/contest/practice-problem-urban-sound-classification/). + +After logging in, the data set can be downloaded. +The details of the dataset and the link to download it are given below: + + +## Urban Sounds Dataset: +### Description + The dataset contains 8732 wav files which are audio samples(<= 4s)) of street sounds like engine_idling, car_horn, children_playing, dog_barking and so on. + The task is to classify these audio samples into one of the following 10 labels: + ``` + siren, + street_music, + drilling, + dog_bark, + children_playing, + gun_shot, + engine_idling, + air_conditioner, + jackhammer, + car_horn + ``` + +To be able to run this example: + +1. `pip install -r requirements.txt` + + If you are in the directory where the requirements.txt file lies, + this step installs the required libraries to run the example. + The main dependency that is required is: Librosa. + The version used to test the example is: `0.6.2` + For more details, refer here: +https://librosa.github.io/librosa/install.html + +2. Download the dataset(train.zip, test.zip) required for this example from the location: +https://drive.google.com/drive/folders/0By0bAi7hOBAFUHVXd1JCN3MwTEU + +3. Extract both the zip archives into the **current directory** - after unzipping you would get 2 new folders namely, + **Train** and **Test** and two csv files - **train.csv**, **test.csv** + + Assuming you are in a directory *"UrbanSounds"*, after downloading and extracting train.zip, the folder structure should be: + + ``` + UrbanSounds + - Train + - 0.wav, 1.wav ... + - train.csv + - train.py + - predict.py ... + ``` + +4. Apache MXNet is installed on the machine. For instructions, go to the link: https://mxnet.incubator.apache.org/install/ + + + +For information on the current design of how the AudioFolderDataset is implemented, refer below: +https://cwiki.apache.org/confluence/display/MXNET/Gluon+-+Audio + +### Usage + +For training: + +- Arguments + - train : The folder/directory that contains the audio(wav) files locally. Default = "./Train" + - csv: The file name of the csv file that contains audio file name to label mapping. Default = "train.csv" + - epochs : Number of epochs to train the model. Default = 30 + - batch_size : The batch size for training. Default = 32 + + +###### To use the default arguments, use: +``` +python train.py +``` +or + +###### To pass command-line arguments for training data directory, epochs, batch_size, csv file name, use : +``` +python train.py --train ./Train --csv train.csv --batch_size 32 --epochs 30 +``` + +For prediction: + +- Arguments + - pred : The folder/directory that contains the audio(wav) files which are to be classified. Default = "./Test" + + +###### To use the default arguments, use: +``` +python predict.py +``` +or + +###### To pass command-line arguments for test data directory, use : +``` +python predict.py --pred ./Test +``` diff --git a/example/gluon/house_prices/README.md b/example/gluon/house_prices/README.md new file mode 100644 index 000000000000..1393a0e3869d --- /dev/null +++ b/example/gluon/house_prices/README.md @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + +# House Prices: Advanced Regression Techniques + +This example shows how to predict house prices and it is based on the [House Price Kaggle challenge](https://www.kaggle.com/c/house-prices-advanced-regression-techniques#description) + +First you need to download train and test data set from here: +``` +https://www.kaggle.com/c/house-prices-advanced-regression-techniques/download/train.csv +https://www.kaggle.com/c/house-prices-advanced-regression-techniques/download/test.csv +``` +Afterwards you can execute the script with ```python kaggle_k_fold_cross_validation.py``` + +For a detailed explanation of the code, you can check out this [chapter](http://d2l.ai/chapter_deep-learning-basics/kaggle-house-price.html) of the Dive into Deep Learning book. diff --git a/example/gluon/lstm_crf/README.md b/example/gluon/lstm_crf/README.md new file mode 100644 index 000000000000..519c3b89f9fd --- /dev/null +++ b/example/gluon/lstm_crf/README.md @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + +# BiLSTM CRF model +This example demonstrates how a [BiLSTM-CRF model](https://arxiv.org/pdf/1508.01991v1.pdf) can be implemented in Gluon to perform noun-phrase chunking as a sequence labeling task. In this example we define the following training sample: +``` +georgia tech is a university in georgia +B I O O O O B +``` +The second line is the IOB representation of the above sentence that is learnt by the model. **I** stands for in chunk, **O** for out of a chunk and **B** for beginning of junks. + +The model consists of an LSTM layer with 2 hidden units and a CRF layer. The CRF layer has a state transition matrix which allows to take past and future tags into account when predicting the current tag. The bidirectional LSTM is reading the word sequence from beginning to end and vice versa. It prodcues a vector representation for the words. The following image is taken from https://arxiv.org/pdf/1508.01991v1.pdf and shows the model architecture: + +![Image taken from https://arxiv.org/pdf/1508.01991v1.pdf](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gluon/lstm_crf/bi-lstm_crf.png) + +You can run the example by executing +``` +python lstm_crf.py +``` +The example code does not take any commandline arguments. If you want to change the number of hidden units or the size of vectors embeddings, then you need to change the variables ```EMBEDDING_DIM``` and ```HIDDEN_DIM```. + + diff --git a/example/gluon/mnist/README.md b/example/gluon/mnist/README.md new file mode 100644 index 000000000000..c053364fad3c --- /dev/null +++ b/example/gluon/mnist/README.md @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + +# MNIST classification example + +This script shows a simple example how to do image classification with Gluon. +The model is trained on MNIST digits image dataset and the goal is to classify the digits ```0-9```. The model has the following layout: +``` +net = nn.Sequential() +with net.name_scope(): + net.add(nn.Dense(128, activation='relu')) + net.add(nn.Dense(64, activation='relu')) + net.add(nn.Dense(10)) +``` + +The script provides the following commandline arguments: + + +``` +MXNet Gluon MNIST Example + +optional arguments: + -h, --help show this help message and exit + --batch-size BATCH_SIZE + batch size for training and testing (default: 100) + --epochs EPOCHS number of epochs to train (default: 10) + --lr LR learning rate (default: 0.1) + --momentum MOMENTUM SGD momentum (default: 0.9) + --cuda Train on GPU with CUDA + --log-interval N how many batches to wait before logging training + status +``` + +After one epoch we get the following output vector for the given test image: + + + +[-5.461655 -4.745 -1.8203478 -0.5705207 8.923972 -2.2358544 -3.3020825 -2.409004 4.0074944 10.362008] + +As we can see the highest activation is 10.362 which corresponds to label `9`. + diff --git a/example/gluon/super_resolution/README.md b/example/gluon/super_resolution/README.md new file mode 100644 index 000000000000..ddcbe8b0a202 --- /dev/null +++ b/example/gluon/super_resolution/README.md @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + +# Superresolution + +This example trains a convolutional neural network to enhance the resolution of images (also known as superresolution). +The script takes the following commandline arguments: + +``` +Super-resolution using an efficient sub-pixel convolution neural network. + +optional arguments: + -h, --help show this help message and exit + --upscale_factor UPSCALE_FACTOR + super resolution upscale factor. default is 3. + --batch_size BATCH_SIZE + training batch size, per device. default is 4. + --test_batch_size TEST_BATCH_SIZE + test batch size + --epochs EPOCHS number of training epochs + --lr LR learning Rate. default is 0.001. + --use-gpu whether to use GPU. + --seed SEED random seed to use. Default=123 + --resolve_img RESOLVE_IMG + input image to use +``` + +Once the network is trained you can use the following command to increase the resolution of your image: +``` +python super_resolution.py --resolve_img myimage.jpg +``` diff --git a/example/gluon/data.py b/example/gluon/super_resolution/data.py similarity index 100% rename from example/gluon/data.py rename to example/gluon/super_resolution/data.py diff --git a/example/model-parallel/README.md b/example/model-parallel/README.md new file mode 100644 index 000000000000..537562070a62 --- /dev/null +++ b/example/model-parallel/README.md @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + +# Run parts of a model on different devices + +This folder contains the example [matrix_factorization](https://github.com/apache/incubator-mxnet/tree/master/example/model-parallel/matrix_factorization) that demonstrates the basic usage of `group2ctxs`. diff --git a/example/sparse/README.md b/example/sparse/README.md new file mode 100644 index 000000000000..8f1302950d22 --- /dev/null +++ b/example/sparse/README.md @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + +# Examples using Sparse Symbol API +This folder contains examples that demonstrate the usage of [Sparse Symbol API](https://mxnet.incubator.apache.org/api/python/symbol/sparse.html) +- [Factorization Machine](https://github.com/apache/incubator-mxnet/tree/master/example/sparse/factorization_machine) uses sparse weights +- [Linear Classification Using Sparse Matrix Multiplication](https://github.com/apache/incubator-mxnet/tree/master/example/sparse/linear_classification) shows how to use a sparse data loader, sparse dot operator and sparse gradient updaters +- [Matrix Factorization w/ Sparse Embedding](https://github.com/apache/incubator-mxnet/tree/master/example/sparse/matrix_factorization) uses sparse weights +- [Wide and Deep Learning](https://github.com/apache/incubator-mxnet/tree/master/example/sparse/wide_deep) shows how to run sparse wide and deep classification + From 349803ce9e737248ef8eb97914fcd87d9a5d75d8 Mon Sep 17 00:00:00 2001 From: Haibin Lin Date: Tue, 19 Feb 2019 16:02:00 -0800 Subject: [PATCH 47/75] Multi-precision AdamW update op (#14171) * mp adamw update * Softmax fp16 (#201) * softmax for fp16 with fp32 accumulator * return AType in kernel * add dtype * kernel * adamw with nan check * add doc * Revert "Softmax fp16 (#201)" This reverts commit 5869e0ae832437c839bb4ccbcc434971bf5c3486. * add test * more test for fp16 * skip update for rescale = 0 --- src/operator/contrib/adamw-inl.h | 165 ++++++++++++++---- src/operator/contrib/adamw.cc | 76 +++++++- src/operator/contrib/adamw.cu | 27 ++- .../python/unittest/test_contrib_optimizer.py | 84 +++++++++ 4 files changed, 310 insertions(+), 42 deletions(-) diff --git a/src/operator/contrib/adamw-inl.h b/src/operator/contrib/adamw-inl.h index 3d76b33ae765..66bd4f3f3ba4 100644 --- a/src/operator/contrib/adamw-inl.h +++ b/src/operator/contrib/adamw-inl.h @@ -33,6 +33,7 @@ #include #include #include +#include #include "../operator_common.h" #include "../mshadow_op.h" #include "../elemwise_op_common.h" @@ -48,7 +49,6 @@ struct AdamWParam : public dmlc::Parameter { float epsilon; float wd; float eta; - float rescale_grad; float clip_gradient; DMLC_DECLARE_PARAMETER(AdamWParam) { DMLC_DECLARE_FIELD(lr) @@ -69,9 +69,6 @@ struct AdamWParam : public dmlc::Parameter { "The penalty scales with the square of the magnitude of each weight."); DMLC_DECLARE_FIELD(eta) .describe("Learning rate schedule multiplier"); - DMLC_DECLARE_FIELD(rescale_grad) - .set_default(1.0f) - .describe("Rescale gradient to grad = rescale_grad*grad."); DMLC_DECLARE_FIELD(clip_gradient) .set_default(-1.0f) .describe("Clip gradient to the range of [-clip_gradient, clip_gradient] " @@ -80,44 +77,138 @@ struct AdamWParam : public dmlc::Parameter { } }; +// rescale_grad is a reserved argument at position -1. Example: +// n_in = 2: weight, grad (fp16) +// n_out = 1: weight (fp16) +// total_in = 6: weight, grad, mean, var, weight32, rescale_grad (fp32) +template +inline bool MPUpdateInferShape(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), static_cast(total_in)) << " in operator " << attrs.name; + CHECK_EQ(out_attrs->size(), static_cast(n_out)) << " in operator " << attrs.name; + // rescale_grad.shape = (1,) + SHAPE_ASSIGN_CHECK(*in_attrs, total_in - 1, mshadow::Shape1(1)); + return ElemwiseAttr( + attrs, in_attrs, out_attrs, TShape()); +} + +// rescale_grad is a reserved argument at position -1. Example: +// n_in = 2: weight, grad (fp16) +// n_out = 1: weight (fp16) +// total_in = 6: weight, grad, mean, var, weight32, rescale_grad (fp32) +template +inline bool MPUpdateInferType(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), static_cast(total_in)) << " in operator " << attrs.name; + CHECK_EQ(out_attrs->size(), static_cast(n_out)) << " in operator " << attrs.name; + for (int i = n_in; i < total_in; ++i) { + TYPE_ASSIGN_CHECK(*in_attrs, i, mshadow::kFloat32); + } + return ElemwiseAttr( + attrs, in_attrs, out_attrs, -1); +} + +template +struct MPAdamWKernel { + template + MSHADOW_XINLINE static void Map(int i, DType* out_data, float* mean_data, + float* var_data, const DType* weight_data, const DType* grad_data, float* weight32, + const float param_clip_gradient, const float param_beta1, const float param_beta2, + const float param_eta, const float param_lr, const float param_wd, + const float param_rescale_grad, const float param_epsilon) { + float w = weight32[i]; + float mean = mean_data[i]; + float var = var_data[i]; + float scaled_grad = param_rescale_grad*static_cast(grad_data[i]); + if (param_clip_gradient >= 0.0f) { + mean = param_beta1 * mean + + (1 - param_beta1) * mshadow_op::clip::Map(scaled_grad, param_clip_gradient); + var = param_beta2 * var + (1 - param_beta2) * + mshadow_op::square::Map(mshadow_op::clip::Map(scaled_grad, param_clip_gradient)); + } else { + mean = param_beta1 * mean + (1 - param_beta1) * scaled_grad; + var = param_beta2 * var + (1 - param_beta2) * mshadow_op::square::Map(scaled_grad); + } + mean_data[i] = mean; + var_data[i] = var; + w = w - param_eta * (param_lr * mean / (mshadow_op::square_root::Map(var) + param_epsilon) + + param_wd * w); + weight32[i] = w; + KERNEL_ASSIGN(out_data[i], req, w); + } +}; + + +template +struct MPAdamWUpdate { + static inline void Forward(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs, + const float rescale_grad) { + using namespace mxnet_op; + AdamWParam param = nnvm::get(attrs.parsed); + Stream* s = ctx.get_stream(); + MSHADOW_REAL_TYPE_SWITCH(inputs[0].type_flag_, DType, { + Tensor weight = inputs[0].FlatTo2D(s); + Tensor grad = inputs[1].FlatTo2D(s); + Tensor mean = inputs[2].FlatTo2D(s); + Tensor var = inputs[3].FlatTo2D(s); + Tensor weight32 = inputs[4].FlatTo2D(s); + Tensor out = outputs[0].FlatTo2D(s); + MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, { + Kernel, xpu>::Launch(s, weight.shape_.Size(), out.dptr_, mean.dptr_, + var.dptr_, weight.dptr_, grad.dptr_, weight32.dptr_, param.clip_gradient, param.beta1, + param.beta2, param.eta, param.lr, param.wd, rescale_grad, param.epsilon); + }); + }); + } +}; + /* * \brief adam_w update. */ template -inline void AdamWUpdate(const nnvm::NodeAttrs& attrs, - const OpContext &ctx, - const std::vector &inputs, - const std::vector &req, - const std::vector &outputs) { - using namespace mshadow; - using namespace mshadow::expr; - using namespace mshadow_op; - const AdamWParam& param = nnvm::get(attrs.parsed); - Stream* s = ctx.get_stream(); - MSHADOW_REAL_TYPE_SWITCH(inputs[0].type_flag_, DType, { - Tensor weight = inputs[0].FlatTo2D(s); - Tensor grad = inputs[1].FlatTo2D(s); - Tensor mean = inputs[2].FlatTo2D(s); - Tensor var = inputs[3].FlatTo2D(s); - Tensor out = outputs[0].FlatTo2D(s); +struct AdamWUpdate { + static inline void Forward(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs, + const float rescale_grad) { + using namespace mshadow; + using namespace mshadow::expr; + using namespace mshadow_op; + const AdamWParam& param = nnvm::get(attrs.parsed); + Stream* s = ctx.get_stream(); + MSHADOW_REAL_TYPE_SWITCH(inputs[0].type_flag_, DType, { + Tensor weight = inputs[0].FlatTo2D(s); + Tensor grad = inputs[1].FlatTo2D(s); + Tensor mean = inputs[2].FlatTo2D(s); + Tensor var = inputs[3].FlatTo2D(s); + Tensor out = outputs[0].FlatTo2D(s); - grad = scalar(param.rescale_grad) * grad; - if (param.clip_gradient >= 0.0f) { - mean = scalar(param.beta1)*mean + scalar(1.f-param.beta1) * - F(grad, DType(param.clip_gradient)); - var = scalar(param.beta2)*var + scalar(1.f-param.beta2)*F( - F(grad, DType(param.clip_gradient))); - } else { - mean = scalar(param.beta1)*mean + scalar(1.f-param.beta1) * grad; - var = scalar(param.beta2)*var + scalar(1.f-param.beta2) * F(grad); - } - Assign(out, req[0], - weight - - scalar(param.eta) * (scalar(param.lr) * - mean / (F(var) + scalar(param.epsilon)) + - (scalar(param.wd) * weight))); - }); -} + grad = scalar(rescale_grad) * grad; + if (param.clip_gradient >= 0.0f) { + mean = scalar(param.beta1)*mean + scalar(1.f-param.beta1) * + F(grad, DType(param.clip_gradient)); + var = scalar(param.beta2)*var + scalar(1.f-param.beta2)*F( + F(grad, DType(param.clip_gradient))); + } else { + mean = scalar(param.beta1)*mean + scalar(1.f-param.beta1) * grad; + var = scalar(param.beta2)*var + scalar(1.f-param.beta2) * F(grad); + } + Assign(out, req[0], + weight - + scalar(param.eta) * (scalar(param.lr) * + mean / (F(var) + scalar(param.epsilon)) + + (scalar(param.wd) * weight))); + }); + } +}; } // namespace op } // namespace mxnet diff --git a/src/operator/contrib/adamw.cc b/src/operator/contrib/adamw.cc index 94623fe08a9e..2fbc39743c93 100644 --- a/src/operator/contrib/adamw.cc +++ b/src/operator/contrib/adamw.cc @@ -24,12 +24,76 @@ * \author Haibin Lin */ #include "./adamw-inl.h" +#include "../optimizer_op-inl.h" namespace mxnet { namespace op { DMLC_REGISTER_PARAMETER(AdamWParam); +template